From 578086290b30071e2a58ac0323fae23bc06e7ffc Mon Sep 17 00:00:00 2001 From: Debmalya Pramanik Date: Tue, 19 Oct 2021 16:29:32 +0530 Subject: [PATCH] bump code v0.1.0-beta * production testing * known issue #4 --- VisualCrossing/api.py | 65 ++- ...Getting Started with Visual Crossing.ipynb | 442 ++++++++++++------ examples/config.json | 28 ++ 3 files changed, 369 insertions(+), 166 deletions(-) create mode 100644 examples/config.json diff --git a/VisualCrossing/api.py b/VisualCrossing/api.py index ee5eb16..69adcb4 100644 --- a/VisualCrossing/api.py +++ b/VisualCrossing/api.py @@ -5,8 +5,10 @@ import urllib import requests import warnings +import numpy as np import pandas as pd from io import StringIO +from copy import deepcopy from datetime import datetime as dt from requests.exceptions import SSLError @@ -63,7 +65,7 @@ def __init__( # default constructor values self.date = date self.APIKey = APIKey - self._location = location + self._location = location if type(location) == str else iter(location) # define keyword arguments self.endDate = kwargs.get("endDate", None) @@ -93,15 +95,15 @@ def __init__( # ) - @property - def BaseURL(self) -> str: + # @property + def BaseURL(self, location : str) -> str: """Base URL for fetching weather data. Check Visual-Crossing documentation for information.""" return "https://weather.visualcrossing.com/VisualCrossingWebServices/rest/services/weatherdata/" + \ self.queryType() + \ - "&location=" + urllib.parse.quote(self._location) + \ + "&location=" + urllib.parse.quote(location) + \ "&key=" + self.APIKey + \ - f"&contentType={self.contentType}" + f"&contentType=csv" # for now, always get as csv def queryType(self) -> str: @@ -129,26 +131,55 @@ def queryType(self) -> str: "&dayStartTime=0:0:00&dayEndTime=23:59:59" + queryDate - def get(self, type : str = pd.DataFrame, **kwargs): + def _get_df_converted_data(self, loc : str, ssl_verify : bool): + """Retreive weather data, and convert it into a pandas dataframe""" + + link = self.BaseURL(loc) + + try: + response = requests.get(link, verify = ssl_verify) + except SSLError: + pass + + code = response.status_code + _weather_data_df = pd.read_csv(StringIO(response.text)) + + return code, deepcopy(_weather_data_df) + + + def get(self, **kwargs): """Fetch API Data and Return in desirable Format""" ssl_verify = kwargs.get("ssl_verify", False) - if ssl_verify: + if not ssl_verify: warnings.warn("using `ssl_verify = True`", VerificationWarning) + + # try: + if type(self._location) != str: + total = np.array([self._get_df_converted_data(loc, ssl_verify) for loc in self._location]) + code = set(total[:, 0]) + code = 200 if len(code) == 1 and list(code)[0] == 200 else list(code.discard(200))[0] + data = pd.concat(total[:, 1]) + del total # house keeping - as pandas holds object into memory + else: + code, data = self._get_df_converted_data(self._location, ssl_verify) + # except SSLError as err: + # # raise ValueError(f"Failed for {err}. If you understand the risk, and want to continue, set `ssl_verify = False`") + # pass - try: - data = requests.get(self.BaseURL, verify = ssl_verify) - except SSLError as err: - # raise ValueError(f"Failed for {err}. If you understand the risk, and want to continue, set `ssl_verify = False`") - pass - - if data.status_code != 200: + if code != 200: raise NoDataFetched(f"unable to fetch any data, recevied code {data.status_code}") if self.contentType == "csv": - data = pd.read_csv(StringIO(data.text)) + data = data.to_csv(index = False) + # data = pd.read_csv(StringIO(data.text)) + elif self.contentType == "dataframe": + pass # return data as is else: # data type is JSON - data = data.json() + # currently, the DataFrame is directly converted into JSON + # with `.to_json()` function instead of the `data.json()` as + # obtained from the API + data = data.to_json() - return data + return deepcopy(data) diff --git a/examples/Getting Started with Visual Crossing.ipynb b/examples/Getting Started with Visual Crossing.ipynb index 69cf4a0..5e47fae 100644 --- a/examples/Getting Started with Visual Crossing.ipynb +++ b/examples/Getting Started with Visual Crossing.ipynb @@ -2,11 +2,11 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "metadata": { "ExecuteTime": { - "end_time": "2021-10-01T08:27:46.568303Z", - "start_time": "2021-10-01T08:27:41.936251Z" + "end_time": "2021-10-19T10:56:05.636550Z", + "start_time": "2021-10-19T10:56:05.315187Z" } }, "outputs": [], @@ -17,46 +17,270 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "metadata": { "ExecuteTime": { - "end_time": "2021-10-01T08:27:46.693474Z", - "start_time": "2021-10-01T08:27:46.601703Z" + "end_time": "2021-10-19T10:56:05.659671Z", + "start_time": "2021-10-19T10:56:05.639110Z" } }, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "from VisualCrossing import API" + "# setting the environment\n", + "# if required, pip install python-dotenv\n", + "from dotenv import load_dotenv # Python 3.6+\n", + "load_dotenv(verbose = True) # configure .env File or set Environment Variables" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ + "

VisualCrossing is a simple to use package to retreive data weather data from VisualCrossing (website name same as that of the package). The API is so configured, that just 4-5 lines of code is sufficient to retreive data easily. The website features both historic as well as report on weather forecast. For more information, check documentation for the same.

\n", + "\n", + "

This document is designed to provide a quick start guide for anyone who wants to test the code. Currently, the code is in Development Status :: 4 - Beta, thus obviously code and functionalities will be upgraded with new features. Note: minimal documentation is currently provided, and will be updated in time.

\n", + "\n", + "Get started by `pip install VisualCrossing`, and then import the package as:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-19T10:56:05.741333Z", + "start_time": "2021-10-19T10:56:05.664416Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0.1.0-beta\n" + ] + } + ], + "source": [ + "import VisualCrossing as vc\n", + "\n", + "# get installed version\n", + "print(vc.__version__)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "ExecuteTime": { + "end_time": "2021-10-19T10:25:29.303119Z", + "start_time": "2021-10-19T10:25:29.290595Z" + }, + "collapsed": true, + "deletable": false, + "editable": false, + "run_control": { + "frozen": true + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Help on class API in module VisualCrossing.api:\n", + "\n", + "class API(VisualCrossing.base)\n", + " | API(date: str, APIKey: str, location: str, **kwargs)\n", + " | \n", + " | A python wrapper to fetch weather data from VisualCrossing\n", + " | \n", + " | The function is devised keeping in mind - minimal coding approach, and recurrent job-scheduling\n", + " | for any person. All the keyword arguments that the API accepts are same as that available\n", + " | in the Visual Crossing API Documentation (check README for more information). The API is configured\n", + " | with default values, which is easier to understand and retreive. However, the data requires certain\n", + " | required arguments defined below. The same can also be obtained from `config.json` file, but is not\n", + " | recomended.\n", + " | \n", + " | :param date: Date for which weather data is required. Pass the date in `YYYY-MM-DD` format,\n", + " | or pass `FORCAST` to get the forecasted data for a particular date.\n", + " | \n", + " | :param APIKey: All VisualCrossing API calls are made with an API key associated with your billing/free\n", + " | account. Get yourself a new Key, if you do not have one. It is advised that the key\n", + " | is not written to a configuration file, and suitable `environment variables` should\n", + " | be called/defined. However, there is a dedicated function :func:`get_key_from_config`\n", + " | which can be used to get `APIKey` from configuration file, which should be defined\n", + " | under `__api_key__`. The key can also be written into file either passing `key` or\n", + " | `__api_key__` as a keyword argument to :func:`generate_config`. # TODO\n", + " | \n", + " | :param location: Location of which the weather is required. Defaults to either a place name (like `india`),\n", + " | or, you can directly pass the coordinates of the particular place as (like `(long, lat)`),\n", + " | or, you can also pass a list/set of locations (either name or coordinates.) # TODO\n", + " | \n", + " | :Keyword Arguments:\n", + " | * *endDate* (``str``) -- When end date is defined, the api fetches data for a given date\n", + " | range which starts from :param:`date` to `endDate`, else only singe day data is fetched.\n", + " | By default, only a single day data is fetched by setting start and end date as :param:`date`.\n", + " | \n", + " | * *unitGroup* (``str``) -- specify unit group, defaults to metric.\n", + " | \n", + " | * *contentType* (``str``) -- specify content type, defaults to csv.\n", + " | \n", + " | * *aggregateHours* (``str``) -- specify aggregate hours, defaults to 24 (daily).\n", + " | \n", + " | Method resolution order:\n", + " | API\n", + " | VisualCrossing.base\n", + " | builtins.object\n", + " | \n", + " | Methods defined here:\n", + " | \n", + " | BaseURL(self, location: str) -> str\n", + " | Base URL for fetching weather data. Check Visual-Crossing documentation for information.\n", + " | \n", + " | __init__(self, date: str, APIKey: str, location: str, **kwargs)\n", + " | Initialize self. See help(type(self)) for accurate signature.\n", + " | \n", + " | get(self, **kwargs)\n", + " | Fetch API Data and Return in desirable Format\n", + " | \n", + " | queryType(self) -> str\n", + " | Set data query type to either `historical` or `forecast` as per given date choice\n", + " | \n", + " | ----------------------------------------------------------------------\n", + " | Methods inherited from VisualCrossing.base:\n", + " | \n", + " | __args_default__(self) -> dict\n", + " | Defines a Dictionary of Default Values for Keyword Arguments (or Attributes)\n", + " | \n", + " | __get_args_default__(self, args: str)\n", + " | Get the Default Value associated with a Keyword Argument\n", + " | \n", + " | __set_attr__(self, response: dict, **kwargs)\n", + " | \n", + " | generate_config(self, defaultSettings: bool = True, fileName: str = 'config.json', overwrite: bool = False, keepBackup: bool = True, **kwargs) -> bool\n", + " | Generate configuration file at `__homepath__` when executed\n", + " | \n", + " | The configuration file can be generated with default settings as defined at\n", + " | :func:`__args_default__` else, user is requested to pass all necessary settings\n", + " | in a correct format (as required by API) to the function, setting `key` as the\n", + " | attribute name, and `value` as the desired value. Users are also advised not to\n", + " | save the `API_KEY` in the configuration file (for security purpose), and to use\n", + " | :func:`_generate_key` to save the key file in an encrypted format.\n", + " | \n", + " | :param defaultSettings: Should you wish to save the configuration file with\n", + " | the default settings. If set to `False` then user is\n", + " | requested to pass all necessary attributes (`key`) and\n", + " | their values. Defaults to `True`.\n", + " | \n", + " | :param fileName: Output file name (with extension - `json`). Defaults to\n", + " | `config.json`.\n", + " | \n", + " | :param overwrite: Overwrite existing configuration file, if exists (same filename).\n", + " | Defaults to `False`.\n", + " | \n", + " | :param keepBackup: If same file name exists, then setting the parameter to `True` will\n", + " | create a backup of the file with the following format\n", + " | `..json` where `UUID` is a randomly generated\n", + " | 7-charecters long name. Defaults to `True`.\n", + " | \n", + " | Accepts n-Keyword Arguments, which are all default settings that can be used to initialize\n", + " | the API.\n", + " | \n", + " | get_key_from_config(self)\n", + " | \n", + " | set_config(self, file: str, **kwargs)\n", + " | \n", + " | ----------------------------------------------------------------------\n", + " | Readonly properties inherited from VisualCrossing.base:\n", + " | \n", + " | __optional_args__\n", + " | Get List of all the Optional Keyword Arguments Accepted by the API\n", + " | \n", + " | ----------------------------------------------------------------------\n", + " | Data descriptors inherited from VisualCrossing.base:\n", + " | \n", + " | __dict__\n", + " | dictionary for instance variables (if defined)\n", + " | \n", + " | __weakref__\n", + " | list of weak references to the object (if defined)\n", + "\n" + ] + } + ], + "source": [ + "help(vc.API) # check documentation" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "

The API can be configured with a configuration which is located at HOME/.visualCrossing/config.json, and an example file is provided in the directory. The file is set, such that same setting can be reused, or if required you can keep pre-defined templates to be used regularly.

\n", + "\n", + "All the attributes required, can be defined like:\n", + "\n", + "```json\n", + "...\n", + " \"attributes\": {\n", + " \"unitGroup\": \"metric\",\n", + " \"contentType\": \"dataframe\",\n", + " \"aggregateHours\": 24\n", + " },\n", + "...\n", + "```\n", + "\n", + "The API requires three minimal options:\n", + "1. Date for which the data is required, in `YYYY-MM-DD` format,\n", + "2. API Key which can be obtained from [website](https://www.visualcrossing.com/weather-api), and\n", + "3. Location for which data is required. The location can be of the following types:\n", + " * Name of a single place, like `\"india\"`,\n", + " * Coordinates of a place (if data for the coordinate is not available, then nearest location data is returned), this should be in `\"latitude,longitude\"` format,\n", + " * To retreive location for multiple location, pass the data like:\n", + " ```python\n", + " [\"india\", \"tokyo\"] # either passing names,\n", + " [\"20.593683,78.962883\", \"35.689487,139.691711\"] # passing list/tuple of coordinates,\n", + " (\"india\", \"35.689487,139.691711\") # or even a combination of name or coordinate\n", + " ```\n", + " * _you can also use US ZIP Codes_.\n", + "\n", "Location is also available by `(lat, lon)` pairs, for which you can also use [this website](https://www.latlong.net/) to find location easily." ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 5, "metadata": { "ExecuteTime": { - "end_time": "2021-10-01T08:28:24.751189Z", - "start_time": "2021-10-01T08:28:24.748189Z" + "end_time": "2021-10-19T10:56:05.749405Z", + "start_time": "2021-10-19T10:56:05.744026Z" } }, "outputs": [], "source": [ - "api = API(\"2021-09-25\", APIKey = \"\", location = \"india\", contentType = \"csv\", aggregateHours = 1) # by default contentType = \"csv\" returns a dataframe" + "locations = (\"india\", \"tokyo\")\n", + "api = vc.API(\"2021-09-25\", APIKey = os.getenv(\"__api_key__\"), location = locations, contentType = \"dataframe\") # default parameters are set from `config.json`" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 6, "metadata": { "ExecuteTime": { - "end_time": "2021-10-01T08:28:34.821460Z", - "start_time": "2021-10-01T08:28:33.358946Z" + "end_time": "2021-10-19T10:56:08.273752Z", + "start_time": "2021-10-19T10:56:05.753306Z" } }, "outputs": [ @@ -64,8 +288,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "D:\\Anaconda3\\lib\\site-packages\\urllib3\\connectionpool.py:979: InsecureRequestWarning: Unverified HTTPS request is being made to host 'weather.visualcrossing.com'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n", - " warnings.warn(\n" + "../VisualCrossing/api.py:160: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.\n", + " total = np.array([self._get_df_converted_data(loc, ssl_verify) for loc in self._location])\n" ] }, { @@ -116,167 +340,87 @@ " \n", " 0\n", " india\n", - " 09/25/2021 00:00:00\n", - " 27.0\n", - " 27.0\n", - " 27.0\n", - " 26.0\n", - " 94.27\n", - " 31.0\n", - " 7.6\n", - " NaN\n", - " ...\n", - " 4.0\n", - " 88.0\n", - " 1008.0\n", - " NaN\n", - " 28.631\n", - " 77.2172\n", - " India\n", - " India\n", - " NaN\n", - " Overcast\n", - " \n", - " \n", - " 1\n", - " india\n", - " 09/25/2021 01:00:00\n", - " 26.0\n", - " 26.0\n", - " 26.0\n", - " 25.0\n", - " 94.23\n", - " NaN\n", - " 7.6\n", - " NaN\n", - " ...\n", - " 4.0\n", - " 88.0\n", - " 1008.0\n", - " NaN\n", - " 28.631\n", - " 77.2172\n", - " India\n", - " India\n", - " NaN\n", - " Overcast\n", - " \n", - " \n", - " 2\n", - " india\n", - " 09/25/2021 02:00:00\n", - " 26.0\n", - " 26.0\n", - " 26.0\n", - " 25.0\n", - " 94.23\n", - " NaN\n", - " 7.6\n", - " NaN\n", - " ...\n", - " 4.0\n", - " 88.0\n", - " 1008.0\n", - " NaN\n", - " 28.631\n", - " 77.2172\n", - " India\n", - " India\n", - " NaN\n", - " Overcast\n", - " \n", - " \n", - " 3\n", - " india\n", - " 09/25/2021 03:00:00\n", - " 26.0\n", - " 26.0\n", - " 26.0\n", - " 25.0\n", - " 94.23\n", - " NaN\n", - " 7.6\n", + " 09/25/2021\n", + " 25.5\n", + " 32.5\n", + " 28.7\n", + " 25.7\n", + " 84.75\n", + " 40.9\n", + " 9.4\n", " NaN\n", " ...\n", - " 4.0\n", - " 88.0\n", - " 1008.0\n", - " NaN\n", - " 28.631\n", + " 3.2\n", + " 69.5\n", + " 1006.3\n", + " Mist, Thunderstorm Without Precipitation, Smok...\n", + " 28.6310\n", " 77.2172\n", " India\n", " India\n", " NaN\n", - " Overcast\n", + " Rain, Partially cloudy\n", " \n", " \n", - " 4\n", - " india\n", - " 09/25/2021 04:00:00\n", - " 26.0\n", - " 26.0\n", - " 26.0\n", - " 25.0\n", - " 94.23\n", + " 0\n", + " tokyo\n", + " 09/25/2021\n", + " 20.4\n", + " 24.1\n", + " 22.2\n", + " 17.2\n", + " 73.29\n", " NaN\n", - " 0.0\n", + " 18.1\n", " NaN\n", " ...\n", - " 3.0\n", - " 50.0\n", - " 1007.0\n", - " NaN\n", - " 28.631\n", - " 77.2172\n", - " India\n", - " India\n", + " 18.8\n", + " 25.4\n", + " 1024.0\n", + " Rain, Light Rain, Sky Coverage Increasing, Smo...\n", + " 35.6841\n", + " 139.8090\n", + " Tokyo, Japan\n", + " Tokyo, Japan\n", " NaN\n", " Partially cloudy\n", " \n", " \n", "\n", - "

5 rows × 25 columns

\n", + "

2 rows × 25 columns

\n", "" ], "text/plain": [ - " Address Date time Minimum Temperature Maximum Temperature \\\n", - "0 india 09/25/2021 00:00:00 27.0 27.0 \n", - "1 india 09/25/2021 01:00:00 26.0 26.0 \n", - "2 india 09/25/2021 02:00:00 26.0 26.0 \n", - "3 india 09/25/2021 03:00:00 26.0 26.0 \n", - "4 india 09/25/2021 04:00:00 26.0 26.0 \n", + " Address Date time Minimum Temperature Maximum Temperature Temperature \\\n", + "0 india 09/25/2021 25.5 32.5 28.7 \n", + "0 tokyo 09/25/2021 20.4 24.1 22.2 \n", + "\n", + " Dew Point Relative Humidity Heat Index Wind Speed Wind Gust ... \\\n", + "0 25.7 84.75 40.9 9.4 NaN ... \n", + "0 17.2 73.29 NaN 18.1 NaN ... \n", "\n", - " Temperature Dew Point Relative Humidity Heat Index Wind Speed \\\n", - "0 27.0 26.0 94.27 31.0 7.6 \n", - "1 26.0 25.0 94.23 NaN 7.6 \n", - "2 26.0 25.0 94.23 NaN 7.6 \n", - "3 26.0 25.0 94.23 NaN 7.6 \n", - "4 26.0 25.0 94.23 NaN 0.0 \n", + " Visibility Cloud Cover Sea Level Pressure \\\n", + "0 3.2 69.5 1006.3 \n", + "0 18.8 25.4 1024.0 \n", "\n", - " Wind Gust ... Visibility Cloud Cover Sea Level Pressure Weather Type \\\n", - "0 NaN ... 4.0 88.0 1008.0 NaN \n", - "1 NaN ... 4.0 88.0 1008.0 NaN \n", - "2 NaN ... 4.0 88.0 1008.0 NaN \n", - "3 NaN ... 4.0 88.0 1008.0 NaN \n", - "4 NaN ... 3.0 50.0 1007.0 NaN \n", + " Weather Type Latitude Longitude \\\n", + "0 Mist, Thunderstorm Without Precipitation, Smok... 28.6310 77.2172 \n", + "0 Rain, Light Rain, Sky Coverage Increasing, Smo... 35.6841 139.8090 \n", "\n", - " Latitude Longitude Resolved Address Name Info Conditions \n", - "0 28.631 77.2172 India India NaN Overcast \n", - "1 28.631 77.2172 India India NaN Overcast \n", - "2 28.631 77.2172 India India NaN Overcast \n", - "3 28.631 77.2172 India India NaN Overcast \n", - "4 28.631 77.2172 India India NaN Partially cloudy \n", + " Resolved Address Name Info Conditions \n", + "0 India India NaN Rain, Partially cloudy \n", + "0 Tokyo, Japan Tokyo, Japan NaN Partially cloudy \n", "\n", - "[5 rows x 25 columns]" + "[2 rows x 25 columns]" ] }, - "execution_count": 4, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "data = api.get()\n", + "data = api.get(ssl_verify = True)\n", "data.head()" ] } @@ -298,7 +442,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.3" + "version": "3.8.8" }, "latex_envs": { "LaTeX_envs_menu_present": true, diff --git a/examples/config.json b/examples/config.json new file mode 100644 index 0000000..6957085 --- /dev/null +++ b/examples/config.json @@ -0,0 +1,28 @@ +{ + "__header__": { + "program": "VisualCrossing", + "version": "0.0.2-alpha", + "homepath": "/home/debmalya/.visualCrossing" + }, + "platform": { + "platform": "Linux-5.4.0-88-generic-x86_64-with-glibc2.29", + "architecture": "x86_64", + "version": "#99-Ubuntu SMP Thu Sep 23 17:29:00 UTC 2021", + "system": "Linux", + "processor": "x86_64", + "uname": [ + "Linux", + "d--------", + "5.4.0-88-generic", + "#99-Ubuntu SMP Thu Sep 23 17:29:00 UTC 2021", + "x86_64", + "x86_64" + ] + }, + "attributes": { + "unitGroup": "metric", + "contentType": "dataframe", + "aggregateHours": 24 + }, + "timestamp": "Mon Oct 18 15:35:29 2021" +}