Skip to content
Snippets Groups Projects
Commit 8fa0e5f5 authored by Tucker Gary Siegel's avatar Tucker Gary Siegel
Browse files

Merge branch 'master' into '30_year_normals'

# Conflicts:
#   to_mongo.py
parents fe05857e e3230e29
No related branches found
No related tags found
1 merge request!4added 30 year normals
......@@ -13,10 +13,12 @@ The data in mongodb has the following fields:
* ```max_temps``` - Maximum daily temperature as an array. One element is one day
* ```normal``` - Indicates whether the data is a 30 year normal or not
Go to ```/docs``` to view the Swagger generated API docs
#### API Endpoints
* ```POST /api/:product/:year```
* Required url params: ```product``` and ```year```. Product is the crop (corn and soybean only supported)
* Body requires latitude and longitude
* Required url params: ```product``` and ```year```. Product is the crop (supports corn, soybean, wheat, tomatoes, potatoes, peas, sunflowers, sugar beets, etc.)
* ```product``` must be singular. As in, send "soybean" not "soybeans"
* Body requires ```latitude``` and ```longitude```, ```t_base``` is an optional parameter if a farmer decides to set their own base temperater in fahrenheit
* returns the gdd calculated for that year up to the most recent date of the year. If the year is before the current, the data will cover 01/01 to 12/31. If it is the current year, 01/01 - current date. But the current year is not included
### How to run
......
Gdd = require('./model');
Gdd = require('./model.js');
exports.year_gdd = function (req, res) {
var year = parseInt(req.params.year)
......@@ -28,12 +28,74 @@ exports.year_gdd = function (req, res) {
var t_max = 86
var t_min = 50
if (product == "soybean" || product == "corn") {
t_base = 50
t_max = 86
t_min = 50
errors = []
if (year < 1981 || year > new Date().getFullYear()) {
errors.push({
parameter_error: "year",
message: year.toString() + " is out of bounds for GDD calculations. Must be between 1981 - Current Year"
});
}
if (latitude < 24.083334 || latitude > 49.916668) {
errors.push({
parameter_error: "latitude",
message: latitude.toString() + " is out of bounds for GDD calculations. Must be between 24.083334 - 49.916668"
});
}
if (req.body.hasOwnProperty("t_base")) {
t_base = parseFloat(req.body.t_base);
if (t_base < t_min) {
t_min = t_base;
}
} else {
res.status(404).send(product + " is not available for GDD calculations")
switch (product) {
case "soybean":
case "corn":
case "sunflower":
case "tomato":
case "sugar_beat":
t_base = 50;
break;
case "potato":
t_base = 44.6;
t_min = 44.6; // NEED TO ASK ABOUT MIN AND MAX TEMPS IN DAY. SHOULD T_MIN BE SET EQUAL TO T_BASE IF IT IS LESS THAN T_BASE?
break;
case "wheat":
t_base = 41.9;
t_min = 41.9;
break;
case "peas":
t_base = 41;
t_min = 41;
break;
case "brussels_sprout":
case "parsley":
case "cabbage":
t_base = 32;
t_min = 32;
break;
default:
errors.push({
parameter_error: "product",
message: product + " is not available for GDD calculations"
});
break;
}
}
if (longitude < -125 || longitude > -66.5) {
errors.push({
parameter_error: "longitude",
message: longitude.toString() + " is out of bounds for GDD calculations. Must be between -125.0 - -66.5"
});
}
if (errors.length > 0) {
res.status(400).send({
errors: errors
})
}
Gdd.findOne(query, projection).then(function(data) {
......@@ -44,7 +106,6 @@ exports.year_gdd = function (req, res) {
var min_temp = 0
var max_temp = 0
var gdd = 0
for (var i = 0; i < min_temps.length; i++) {
min_temp = min_temps[i] >= t_min ? min_temps[i] : t_min;
......
This diff is collapsed.
......@@ -12,6 +12,8 @@
"body-parser": "^1.19.0",
"express": "^4.17.1",
"mongoose": "^5.11.15",
"nodemon": "^2.0.7"
"nodemon": "^2.0.7",
"swagger-jsdoc": "5.0.1",
"swagger-ui-express": "^4.1.6"
}
}
import gdal
import numpy as np
import xarray as xr
import random
def to_freedom_units(data):
return (data * 9/5) + 32
def ReadBilFile(bil):
gdal.GetDriverByName('EHdr').Register()
img = gdal.Open(bil)
band = img.GetRasterBand(1)
data = band.ReadAsArray()
return data
if __name__ == '__main__':
import glob
years = list(range(2020, 2020 + 1))
for y in years:
print (y)
all_ = []
print ("data/PRISM/tmin/PRISM_tmin_stable_4kmD2_%s0101_%s1231/*.bil" % (y, y))
for file in sorted(list(glob.glob("data/PRISM/tmin/PRISM_tmin_stable_4kmD2_%s0101_%s1231_bil/*.bil" % (y, y)))):
# print (file)
a = ReadBilFile(file)
a[a == -9999] = np.nan
a = np.flipud(a)
all_.append(a)
all_ = np.stack(all_)
all_ = to_freedom_units(all_)
tmin = xr.Variable(['date', 'row', 'col'], all_, attrs={"long_name": "min temperature in f"}).chunk({"date": -1}).astype(np.dtype(np.float32))
all_ = []
print ("data/PRISM/tmax/PRISM_tmax_stable_4kmD2_%s0101_%s1231/*.bil" % (y, y))
for file in sorted(list(glob.glob("data/PRISM/tmax/PRISM_tmax_stable_4kmD2_%s0101_%s1231_bil/*.bil" % (y, y)))):
# print (file)
a = ReadBilFile(file)
a[a == -9999] = np.nan
a = np.flipud(a)
all_.append(a)
all_ = np.stack(all_)
all_ = to_freedom_units(all_)
tmax = xr.Variable(['date', 'row', 'col'], all_, attrs={"long_name": "max temperature in f"}).chunk({"date": -1}).astype(np.dtype(np.float32))
data = xr.Dataset(
{
"tmax": tmax,
"tmin": tmin
},
)
print (data)
data.to_netcdf("data/temps_%s.nc" % y)
import os, zipfile, tqdm
import gdal, shutil, datetime
import numpy as np
import xarray as xr
from ftplib import FTP
from multiprocessing.pool import ThreadPool
from multiprocessing import Lock
from pymongo import MongoClient
def to_freedom_units(data):
return (data * 9/5) + 32
def read_bil_file(bil):
gdal.GetDriverByName('EHdr').Register()
img = gdal.Open(bil)
band = img.GetRasterBand(1)
data = band.ReadAsArray()
return data
client = MongoClient("mongodb+srv://gdd-server:u8i3icLAJXjZEhTs@cluster0.wdxf4.mongodb.net")
db = client["gdd_database"]
gdd = db.gdd_current
gdd.drop()
gdd = db["gdd_current"]
resp = gdd.create_index([ ("location", "2dsphere") ])
resp = gdd.create_index([ ("year", 1) ])
def process_file(data):
file, lock, data_type = data
ftp = FTP(ftp_url)
ftp.login(user=ftp_user, passwd=ftp_pass)
ftp.cwd("daily")
ftp.cwd(data_type)
ftp.cwd("2021")
base_name = file.split(".")[0]
with open(file, 'wb') as f:
ftp.retrbinary('RETR ' + file, f.write)
ftp.quit()
with zipfile.ZipFile(file, "r") as f:
f.extractall(base_name)
data = read_bil_file("%s/%s.bil" % (base_name, base_name))
data[data == -9999] = np.nan
data = np.flipud(data)
shutil.rmtree(base_name)
os.remove(file)
return (data, base_name.split("_")[4])
ftp_url = "prism.nacse.org"
ftp_user = "anonymous"
ftp_pass = "tgsiegel@umd.edu"
def run(year, dtype):
ftp = FTP(ftp_url)
ftp.login(user=ftp_user, passwd=ftp_pass)
ftp.cwd("daily")
ftp.cwd(dtype)
ftp.cwd(str(year))
files = ftp.nlst()
ftp.quit()
all_data = []
files.sort(key=lambda x: x.split("_")[4])
# files = files[-10:]
lock = Lock()
locks = [lock] * len(files)
types = [dtype] * len(files)
pool = ThreadPool(10)
results = pool.map(process_file, zip(files, locks, types))
data = [r[0] for r in results]
data = to_freedom_units(np.stack(data))
return data
year = 2021
print ("Pulling tmax data")
tmax_data = run(year, "tmax")
print ("Pulling tmin data")
tmin_data = run(year, "tmin")
coords = xr.open_dataset("coords.nc")
lat = coords.latitude.data
lon = coords.longitude.data
soy = np.datetime64("%s-01-01" % year)
x = np.where(~np.isnan(np.nanmean(tmin_data, axis=0)))
lat = lat[::-1]
# FORCE LOCATIONS TO COLLEGE PARK, LAT 38.99 LON -76.94 BECAUSE OF ATLAS LIMIT
a1 = np.where(38 < lat)[0].tolist()
a2 = np.where(lat < 40)[0].tolist()
lat_a = np.array(list(set(a1) & set(a2)))
a1 = np.where(-77 < lon)[0].tolist()
a2 = np.where(lon < -75)[0].tolist()
lon_a = np.array(list(set(a1) & set(a2)))
x1 = np.array(np.meshgrid(lat_a, lon_a)).T.reshape(len(lat_a) * len(lon_a), 2).tolist()
x1 = [(z[0], z[1]) for z in x1]
x2 = [(a, b) for a, b in zip(x[0], x[1])] # fix to x = [..... (x[0], x[1])] and all limiting stuff above and below when atlas limit removed
x = list(set(x1) & set(x2))
tmins = tmin_data
tmaxs = tmax_data
locs = []
print ("uploading to mongo")
count = 0
for i in tqdm.tqdm(x):
if len(locs) % 100 == 0 and len(locs) != 0:
new_result = gdd.insert_many(locs)
locs = []
tmin_ = tmins[:, i[0], i[1]]
tmax_ = tmaxs[:, i[0], i[1]]
lat_ = lat[i[0]]
lon_ = lon[i[1]]
a = i
t = {}
_id = str(year) + "_"
_id += str(a[0]) + "_" + str(a[1])
t["location"] = {"type": "Point", "coordinates": [float(lon_), float(lat_)]}
t["prism_lat"] = int(a[0])
t["prism_lon"] = int(a[1])
t["last_date"] = datetime.datetime.strptime(str(soy + np.timedelta64(len(tmin_) - 1, "D")) , "%Y-%m-%d")
t["year"] = int(year)
t["min_temps"] = list([float(a) for a in tmin_])
t["max_temps"] = list([float(a) for a in tmax_])
t["_id"] = _id
locs.append(t)
count += 1
if len(locs) != 0:
new_result = gdd.insert_many(locs)
\ No newline at end of file
......@@ -2,4 +2,5 @@ xarray
numpy
gdal
tqdm
pymongo
\ No newline at end of file
pymongo
dnspython
\ No newline at end of file
......@@ -2,8 +2,99 @@
let router = require('express').Router();
var gddController = require('./gddController');
/**
* @swagger
* api/{product}/{year}:
* post:
* summary: Returns GDD data
* description: Returns GDD data for a specific product, year, lat, and lon
* parameters:
* - in: path
* name: product
* required: true
* description: Agricultural product to calculate gdd for
* schema:
* type: string
* enum: [corn, soybean, sugar_beet, sunflower, tomato, potato, wheat, peas, parsley, brussels_sprouts, cabbage]
* - in: path
* name: year
* required: true
* description: Year to calculate gdd on
* schema:
* type: integer
* minimum: 1981
* - in: body
* description: Data to calculate gdd on
* schema:
* type: object
* required:
* - longitude
* - latitude
* properties:
* latitude:
* description: latitude to calculate gdd on
* type: number
* minimum: 24.083334
* maximum: 49.916668
* example: 25.6
* longitude:
* description: longitude to calculate gdd on
* type: number
* minimum: -125.0
* maximum: -66.5
* example: -78.5
* t_base:
* description: Base temperature to calculate gdd on, in fahrenheit
* type: number
* example: 50
*
* responses:
* 200:
* description: Success
* content:
* application/json:
* schema:
* type: object
* properties:
* message:
* type: string
* example: GDDs
* date:
* type: string
* format: date
* data:
* type: array
* items:
* type: number
* minItems: 1
* maxItems: 365
* 400:
* description: Bad Request
* content:
* application/json:
* schema:
* type: object
* properties:
* errors:
* type: array
* items:
* type: object
* properties:
* parameter_error:
* type: string
* example: latitude
* message:
* type: string
* example: 22.5 is out of bounds for GDD calculations. Must be between 24.083334 - 49.916668
*
*
*
*
*/
router.route('/:product/:year')
.post(gddController.year_gdd)
module.exports = router;
\ No newline at end of file
module.exports = router;
const express = require('express');
let bodyParser = require('body-parser');
// const express = require('express');
// let bodyParser = require('body-parser');
let mongoose = require('mongoose');
require('./model.js');
const swaggerJSDoc = require('swagger-jsdoc');
const swaggerUi = require('swagger-ui-express');
const port = 3000;
// var swaggerJsdoc = require("swagger-jsdoc");
// var swaggerUi = require("swagger-ui-express");
var express = require("express"),
bodyParser = require("body-parser");
// swaggerUi = require("swagger-ui-express");
require('./model.js');
const port = 4000;
const app = express();
const expressSwagger = require('express-swagger-generator')(app);
let apiRoutes = require("./routes")
//Use API routes in the App
......@@ -17,6 +27,30 @@ const dbPath = 'mongodb://localhost/gdd_database';
const options = {useNewUrlParser: true, useUnifiedTopology: true}
const mongo = mongoose.connect(dbPath, options);
const swaggerDefinition = {
openapi: '3.0.0',
info: {
title: 'Express API for JSONPlaceholder',
version: '1.0.0',
},
servers: [
{
url: 'http://localhost:4000',
description: 'GDD server',
},
],
};
const swagger_options = {
swaggerDefinition,
// Paths to files containing OpenAPI definitions
apis: ['./routes.js'],
};
const swaggerSpec = swaggerJSDoc(swagger_options);
app.use('/docs', swaggerUi.serve, swaggerUi.setup(swaggerSpec));
app.listen(port, function () {
console.log("Server is running on "+ port +" port");
});
......
......@@ -16,7 +16,7 @@ def is_leap_year(year):
else:
return False
client = MongoClient('0.0.0.0', 27017)
client = MongoClient("mongodb+srv://gdd-server:u8i3icLAJXjZEhTs@cluster0.wdxf4.mongodb.net")
db = client["gdd_database"]
......@@ -32,6 +32,7 @@ resp = gdd.create_index([ ("year", 1) ])
coords = xr.open_dataset("coords.nc")
lat = coords.latitude.data
lon = coords.longitude.data
lat = lat[::-1]
years = list(range(1981, 2020 + 1))
......@@ -40,9 +41,22 @@ for year in years:
print (year)
data = xr.open_dataset("data/temps_%s.nc" % year)
x = np.where(~np.isnan(np.nanmean(data.tmin.data, axis=0)))
x = [(a, b) for a, b in zip(x[0], x[1])]
# x = [(a, b) for a, b in zip(lat_a, lon_a)] # fix to x[0], x[1] when atlas limit removed
# FORCE LOCATIONS TO COLLEGE PARK, LAT 38.99 LON -76.94 BECAUSE OF ATLAS LIMIT
a1 = np.where(38.5 < lat)[0].tolist()
a2 = np.where(lat < 39.5)[0].tolist()
lat_a = np.array(list(set(a1) & set(a2)))
a1 = np.where(-77 < lon)[0].tolist()
a2 = np.where(lon < -76)[0].tolist()
lon_a = np.array(list(set(a1) & set(a2)))
lat = lat[::-1]
x1 = np.array(np.meshgrid(lat_a, lon_a)).T.reshape(len(lat_a) * len(lon_a), 2).tolist()
x1 = [(z[0], z[1]) for z in x1]
x2 = [(a, b) for a, b in zip(x[0], x[1])] # fix to x = [..... (x[0], x[1])] and all limiting stuff above and below when atlas limit removed
x = list(set(x1) & set(x2))
tmins = data.tmin.data
tmaxs = data.tmax.data
......@@ -84,7 +98,6 @@ for year in years:
count += 1
### 30 YEAR NORMALS ###
### Covers from 1981-2010 ###
......@@ -161,4 +174,7 @@ for i in tqdm.tqdm(x):
locs.append(t)
count += 1
\ No newline at end of file
count += 1
if len(locs) != 0:
new_result = gdd.insert_many(locs)
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment