Merge branch 'master' into '30_year_normals'

# Conflicts:
#   to_mongo.py

README.md

@@ -13,10 +13,12 @@ The data in mongodb has the following fields:
 * ```max_temps``` - Maximum daily temperature as an array. One element is one day
 * ```normal``` - Indicates whether the data is a 30 year normal or not
 
+Go to ```/docs``` to view the Swagger generated API docs
 #### API Endpoints
 * ```POST /api/:product/:year```
-    * Required url params: ```product``` and ```year```. Product is the crop (corn and soybean only supported)
-    * Body requires latitude and longitude
+    * Required url params: ```product``` and ```year```. Product is the crop (supports corn, soybean, wheat, tomatoes, potatoes, peas, sunflowers, sugar beets, etc.)
+    * ```product``` must be singular. As in, send "soybean" not "soybeans"
+    * Body requires ```latitude``` and ```longitude```, ```t_base``` is an optional parameter if a farmer decides to set their own base temperater in fahrenheit
     * returns the gdd calculated for that year up to the most recent date of the year. If the year is before the current, the data will cover 01/01 to 12/31. If it is the current year, 01/01 - current date. But the current year is not included
 
 ### How to run

gddController.js

-Gdd = require('./model');
+Gdd = require('./model.js');
 
 exports.year_gdd = function (req, res) {
     var year = parseInt(req.params.year)
@@ -28,12 +28,74 @@ exports.year_gdd = function (req, res) {
     var t_max = 86
     var t_min = 50 
 
-    if (product == "soybean" || product == "corn") {
-        t_base = 50
-        t_max = 86
-        t_min = 50 
+    errors = []
+
+    if (year < 1981 || year > new Date().getFullYear()) {
+        errors.push({
+            parameter_error: "year",
+            message: year.toString() + " is out of bounds for GDD calculations. Must be between 1981 - Current Year"
+        });
+    }
+
+    if (latitude < 24.083334 || latitude > 49.916668) {
+        errors.push({
+            parameter_error: "latitude",
+            message: latitude.toString() + " is out of bounds for GDD calculations. Must be between 24.083334 - 49.916668"
+        });
+    }
+
+    if (req.body.hasOwnProperty("t_base")) {
+        t_base = parseFloat(req.body.t_base);
+        if (t_base < t_min) {
+            t_min = t_base;
+        }
     } else {
-        res.status(404).send(product + " is not available for GDD calculations")
+        switch (product) {
+            case "soybean":
+            case "corn":
+            case "sunflower":
+            case "tomato":
+            case "sugar_beat":
+                t_base = 50;
+                break;
+            case "potato":
+                t_base = 44.6;
+                t_min = 44.6; // NEED TO ASK ABOUT MIN AND MAX TEMPS IN DAY. SHOULD T_MIN BE SET EQUAL TO T_BASE IF IT IS LESS THAN T_BASE?
+                break;
+            case "wheat":
+                t_base = 41.9;
+                t_min = 41.9;
+                break;
+            case "peas":
+                t_base = 41;
+                t_min = 41;
+                break;
+            case "brussels_sprout":
+            case "parsley":
+            case "cabbage":
+                t_base = 32;
+                t_min = 32;
+                break;
+            default:
+                errors.push({
+                    parameter_error: "product",
+                    message: product + " is not available for GDD calculations"
+                });
+                break;
+        }
+    }
+
+    if (longitude < -125 || longitude > -66.5) {
+        errors.push({
+            parameter_error: "longitude",
+            message: longitude.toString() + " is out of bounds for GDD calculations. Must be between -125.0 - -66.5"
+        });
+    }
+
+    if (errors.length > 0) {
+        res.status(400).send({
+            errors: errors
+        })
     }
 
     Gdd.findOne(query, projection).then(function(data) {
@@ -44,7 +106,6 @@ exports.year_gdd = function (req, res) {
         
         var min_temp = 0
         var max_temp = 0
-        var gdd = 0
 
         for (var i = 0; i < min_temps.length; i++) {
             min_temp = min_temps[i] >= t_min ? min_temps[i] : t_min;

package.json

@@ -12,6 +12,8 @@
     "body-parser": "^1.19.0",
     "express": "^4.17.1",
     "mongoose": "^5.11.15",
-    "nodemon": "^2.0.7"
+    "nodemon": "^2.0.7",
+    "swagger-jsdoc": "5.0.1",
+    "swagger-ui-express": "^4.1.6"
   }
 }

preprocess.py

-import gdal
-import numpy as np
-import xarray as xr
-import random 
-
-def to_freedom_units(data):
-    return (data * 9/5) + 32 
-
-
-def ReadBilFile(bil):
-
-    gdal.GetDriverByName('EHdr').Register()
-    img = gdal.Open(bil)
-    band = img.GetRasterBand(1)
-    data = band.ReadAsArray()
-    return data
-
-if __name__ == '__main__':
-
-    import glob 
-    years = list(range(2020, 2020 + 1))
-
-
-    for y in years:
-        print (y)
-        all_ = []
-        print ("data/PRISM/tmin/PRISM_tmin_stable_4kmD2_%s0101_%s1231/*.bil" % (y, y))
-        for file in sorted(list(glob.glob("data/PRISM/tmin/PRISM_tmin_stable_4kmD2_%s0101_%s1231_bil/*.bil" % (y, y)))):
-            # print (file)
-            a = ReadBilFile(file)
-            a[a == -9999] = np.nan
-            a = np.flipud(a)
-            all_.append(a)
-
-        all_ = np.stack(all_)
-        all_ = to_freedom_units(all_)
-        tmin = xr.Variable(['date', 'row', 'col'], all_, attrs={"long_name": "min temperature in f"}).chunk({"date": -1}).astype(np.dtype(np.float32))
-        all_ = []
-        print ("data/PRISM/tmax/PRISM_tmax_stable_4kmD2_%s0101_%s1231/*.bil" % (y, y))
-        for file in sorted(list(glob.glob("data/PRISM/tmax/PRISM_tmax_stable_4kmD2_%s0101_%s1231_bil/*.bil" % (y, y)))):
-            # print (file)
-            a = ReadBilFile(file)
-            a[a == -9999] = np.nan
-            a = np.flipud(a)
-            all_.append(a)
-
-        all_ = np.stack(all_)
-        all_ = to_freedom_units(all_)
-        tmax = xr.Variable(['date', 'row', 'col'], all_, attrs={"long_name": "max temperature in f"}).chunk({"date": -1}).astype(np.dtype(np.float32))
-        
-        data = xr.Dataset(
-            {
-                "tmax": tmax,
-                "tmin": tmin
-            }, 
-        )
-        print (data)
-        data.to_netcdf("data/temps_%s.nc" % y)

pull_data.py

+import os, zipfile, tqdm
+import gdal, shutil, datetime
+import numpy as np
+import xarray as xr
+from ftplib import FTP
+from multiprocessing.pool import ThreadPool
+from multiprocessing import Lock
+from pymongo import MongoClient
+
+def to_freedom_units(data):
+    return (data * 9/5) + 32 
+
+def read_bil_file(bil):
+
+    gdal.GetDriverByName('EHdr').Register()
+    img = gdal.Open(bil)
+    band = img.GetRasterBand(1)
+    data = band.ReadAsArray()
+    return data
+
+client = MongoClient("mongodb+srv://gdd-server:u8i3icLAJXjZEhTs@cluster0.wdxf4.mongodb.net")
+
+db = client["gdd_database"]
+
+gdd = db.gdd_current
+
+gdd.drop()
+
+gdd = db["gdd_current"]
+
+resp = gdd.create_index([ ("location", "2dsphere") ])
+resp = gdd.create_index([ ("year", 1) ])
+
+def process_file(data):
+    file, lock, data_type = data
+
+    ftp = FTP(ftp_url)
+    ftp.login(user=ftp_user, passwd=ftp_pass)
+    ftp.cwd("daily")
+    ftp.cwd(data_type)
+    ftp.cwd("2021")
+
+    base_name = file.split(".")[0]
+    
+    with open(file, 'wb') as f:
+        ftp.retrbinary('RETR ' + file, f.write)
+    ftp.quit()
+    
+    with zipfile.ZipFile(file, "r") as f:
+        f.extractall(base_name)
+    
+    data = read_bil_file("%s/%s.bil" % (base_name, base_name))
+
+    data[data == -9999] = np.nan
+    data = np.flipud(data)
+    shutil.rmtree(base_name)
+    os.remove(file)
+
+    return (data, base_name.split("_")[4])
+
+ftp_url = "prism.nacse.org"
+ftp_user = "anonymous"
+ftp_pass = "tgsiegel@umd.edu"
+
+def run(year, dtype):
+
+    ftp = FTP(ftp_url)
+    ftp.login(user=ftp_user, passwd=ftp_pass)
+    ftp.cwd("daily")
+    ftp.cwd(dtype)
+    ftp.cwd(str(year))
+    files = ftp.nlst()
+    ftp.quit()
+
+    all_data = []
+
+    files.sort(key=lambda x: x.split("_")[4])
+    # files = files[-10:]
+    lock = Lock()
+    locks = [lock] * len(files)
+    types = [dtype] * len(files)
+
+    pool = ThreadPool(10)
+
+    results = pool.map(process_file, zip(files, locks, types))
+
+    data = [r[0] for r in results]
+    data = to_freedom_units(np.stack(data))
+    return data 
+
+year = 2021
+print ("Pulling tmax data")
+tmax_data = run(year, "tmax")
+print ("Pulling tmin data")
+tmin_data = run(year, "tmin")
+
+coords = xr.open_dataset("coords.nc")
+lat = coords.latitude.data
+lon = coords.longitude.data
+
+soy = np.datetime64("%s-01-01" % year)
+x = np.where(~np.isnan(np.nanmean(tmin_data, axis=0)))
+lat = lat[::-1]
+
+# FORCE LOCATIONS TO COLLEGE PARK, LAT 38.99 LON -76.94 BECAUSE OF ATLAS LIMIT
+
+a1 = np.where(38 < lat)[0].tolist()
+a2 = np.where(lat < 40)[0].tolist()
+lat_a = np.array(list(set(a1) & set(a2)))
+
+a1 = np.where(-77 < lon)[0].tolist()
+a2 = np.where(lon < -75)[0].tolist()
+lon_a = np.array(list(set(a1) & set(a2)))
+
+x1 = np.array(np.meshgrid(lat_a, lon_a)).T.reshape(len(lat_a) * len(lon_a), 2).tolist()
+x1 = [(z[0], z[1]) for z in x1]
+x2 = [(a, b) for a, b in zip(x[0], x[1])] # fix to x = [..... (x[0], x[1])] and all limiting stuff above and below when atlas limit removed
+
+x = list(set(x1) & set(x2))
+
+
+tmins = tmin_data
+tmaxs = tmax_data
+
+locs = []
+print ("uploading to mongo")
+count = 0
+for i in tqdm.tqdm(x):
+    if len(locs) % 100 == 0 and len(locs) != 0:
+        new_result = gdd.insert_many(locs)
+        locs = []
+
+    tmin_ = tmins[:, i[0], i[1]]
+    tmax_ = tmaxs[:, i[0], i[1]]
+    
+    lat_ = lat[i[0]]
+    lon_ = lon[i[1]]
+
+    a = i
+
+    t = {}
+
+    _id = str(year) + "_"
+
+    _id += str(a[0]) + "_" + str(a[1])
+    
+    t["location"] = {"type": "Point", "coordinates": [float(lon_), float(lat_)]}
+    t["prism_lat"] = int(a[0])
+    t["prism_lon"] = int(a[1])
+    
+    t["last_date"] = datetime.datetime.strptime(str(soy + np.timedelta64(len(tmin_) - 1, "D")) , "%Y-%m-%d")
+    t["year"] = int(year)
+    t["min_temps"] = list([float(a) for a in tmin_])
+    t["max_temps"] = list([float(a) for a in tmax_])
+    t["_id"] = _id
+    
+    locs.append(t)
+    
+    count += 1
+
+if len(locs) != 0:
+    new_result = gdd.insert_many(locs)
\ No newline at end of file

requirements.txt

@@ -2,4 +2,5 @@ xarray
 numpy
 gdal
 tqdm
-pymongo
+pymongo
\ No newline at end of file
+dnspython
\ No newline at end of file

routes.js

@@ -2,8 +2,99 @@
 let router = require('express').Router();
 
 var gddController = require('./gddController');
-
+/**
+ * @swagger
+ * api/{product}/{year}:
+ *      post:
+ *          summary: Returns GDD data
+ *          description: Returns GDD data for a specific product, year, lat, and lon
+ *          parameters:
+ *            - in: path
+ *              name: product
+ *              required: true
+ *              description: Agricultural product to calculate gdd for
+ *              schema:
+ *                  type: string
+ *                  enum: [corn, soybean, sugar_beet, sunflower, tomato, potato, wheat, peas, parsley, brussels_sprouts, cabbage]
+ *            - in: path
+ *              name: year
+ *              required: true
+ *              description: Year to calculate gdd on
+ *              schema:
+ *                  type: integer
+ *                  minimum: 1981
+ *            - in: body
+ *              description: Data to calculate gdd on
+ *              schema:
+ *                  type: object
+ *                  required:
+ *                    - longitude
+ *                    - latitude
+ *                  properties:
+ *                      latitude:
+ *                          description: latitude to calculate gdd on
+ *                          type: number
+ *                          minimum: 24.083334
+ *                          maximum: 49.916668
+ *                          example: 25.6
+ *                      longitude:
+ *                          description: longitude to calculate gdd on
+ *                          type: number
+ *                          minimum: -125.0
+ *                          maximum: -66.5
+ *                          example: -78.5
+ *                      t_base:
+ *                          description: Base temperature to calculate gdd on, in fahrenheit
+ *                          type: number
+ *                          example: 50
+ *                  
+ *          responses:
+ *              200:
+ *                  description: Success
+ *                  content:
+ *                      application/json:
+ *                          schema:
+ *                              type: object
+ *                              properties:
+ *                                  message:
+ *                                      type: string
+ *                                      example: GDDs
+ *                                  date:
+ *                                      type: string
+ *                                      format: date
+ *                                  data:
+ *                                      type: array
+ *                                      items:
+ *                                          type: number
+ *                                      minItems: 1
+ *                                      maxItems: 365
+ *              400:
+ *                  description: Bad Request
+ *                  content:
+ *                      application/json:
+ *                          schema:
+ *                              type: object
+ *                              properties:
+ *                                  errors:
+ *                                      type: array
+ *                                      items:
+ *                                          type: object
+ *                                          properties:
+ *                                              parameter_error:
+ *                                                  type: string
+ *                                                  example: latitude
+ *                                              message:
+ *                                                  type: string
+ *                                                  example: 22.5 is out of bounds for GDD calculations. Must be between 24.083334 - 49.916668
+ * 
+ *                  
+ *                          
+ *          
+*/
 router.route('/:product/:year')
     .post(gddController.year_gdd)
 
-module.exports = router;
+module.exports = router;
\ No newline at end of file
+
+
+

server.js

-const express = require('express');
-let bodyParser = require('body-parser');
+// const express = require('express');
+// let bodyParser = require('body-parser');
 let mongoose = require('mongoose');
-require('./model.js');
+const swaggerJSDoc = require('swagger-jsdoc');
+const swaggerUi = require('swagger-ui-express');
 
-const port = 3000;
+// var swaggerJsdoc = require("swagger-jsdoc");
+// var swaggerUi = require("swagger-ui-express");
+var express = require("express"),
+  bodyParser = require("body-parser");
+
+  //   swaggerUi = require("swagger-ui-express");
+  require('./model.js');
+  
+const port = 4000;
 
 const app = express();
+const expressSwagger  = require('express-swagger-generator')(app);
 
 let apiRoutes = require("./routes")
 //Use API routes in the App
@@ -17,6 +27,30 @@ const dbPath = 'mongodb://localhost/gdd_database';
 const options = {useNewUrlParser: true, useUnifiedTopology: true}
 const mongo = mongoose.connect(dbPath, options);
 
+const swaggerDefinition = {
+    openapi: '3.0.0',
+    info: {
+      title: 'Express API for JSONPlaceholder',
+      version: '1.0.0',
+    },
+    servers: [
+        {
+          url: 'http://localhost:4000',
+          description: 'GDD server',
+        },
+      ],
+  };
+  
+  const swagger_options = {
+    swaggerDefinition,
+    // Paths to files containing OpenAPI definitions
+    apis: ['./routes.js'],
+  };
+
+const swaggerSpec = swaggerJSDoc(swagger_options);
+
+app.use('/docs', swaggerUi.serve, swaggerUi.setup(swaggerSpec));
+
 app.listen(port, function () {
     console.log("Server is running on "+ port +" port");
 });

to_mongo.py

@@ -16,7 +16,7 @@ def is_leap_year(year):
     else:
         return False 
 
-client = MongoClient('0.0.0.0', 27017)
+client = MongoClient("mongodb+srv://gdd-server:u8i3icLAJXjZEhTs@cluster0.wdxf4.mongodb.net")
 
 db = client["gdd_database"]
 
@@ -32,6 +32,7 @@ resp = gdd.create_index([ ("year", 1) ])
 coords = xr.open_dataset("coords.nc")
 lat = coords.latitude.data
 lon = coords.longitude.data
+lat = lat[::-1]
 
 years = list(range(1981, 2020 + 1))
 
@@ -40,9 +41,22 @@ for year in years:
     print (year)
     data = xr.open_dataset("data/temps_%s.nc" % year)
     x = np.where(~np.isnan(np.nanmean(data.tmin.data, axis=0)))
-    x = [(a, b) for a, b in zip(x[0], x[1])]
+    # x = [(a, b) for a, b in zip(lat_a, lon_a)] # fix to x[0], x[1] when atlas limit removed
+    # FORCE LOCATIONS TO COLLEGE PARK, LAT 38.99 LON -76.94 BECAUSE OF ATLAS LIMIT
+
+    a1 = np.where(38.5 < lat)[0].tolist()
+    a2 = np.where(lat < 39.5)[0].tolist()
+    lat_a = np.array(list(set(a1) & set(a2)))
+
+    a1 = np.where(-77 < lon)[0].tolist()
+    a2 = np.where(lon < -76)[0].tolist()
+    lon_a = np.array(list(set(a1) & set(a2)))
 
-    lat = lat[::-1]
+    x1 = np.array(np.meshgrid(lat_a, lon_a)).T.reshape(len(lat_a) * len(lon_a), 2).tolist()
+    x1 = [(z[0], z[1]) for z in x1]
+    x2 = [(a, b) for a, b in zip(x[0], x[1])] # fix to x = [..... (x[0], x[1])] and all limiting stuff above and below when atlas limit removed
+
+    x = list(set(x1) & set(x2))
 
     tmins = data.tmin.data
     tmaxs = data.tmax.data 
@@ -84,7 +98,6 @@ for year in years:
         
         count += 1
 
-
 ### 30 YEAR NORMALS ###
 ### Covers from 1981-2010 ###
 
@@ -161,4 +174,7 @@ for i in tqdm.tqdm(x):
     
     locs.append(t)
     
-    count += 1
+    count += 1
\ No newline at end of file
+
+if len(locs) != 0:
+    new_result = gdd.insert_many(locs)