Skip to content

M
mooc-rr
Commit e79b30ea authored by fcf61da24638f93d44cd057b11a6abdc's avatar fcf61da24638f93d44cd057b11a6abdc
Browse files
Options

initial commit, added sha256 and automatic versionning of file from url

parent 8c63b547
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 2376 additions and 12 deletions
module3/exo1/20251210-sentinelle-syndrome-grippal.csv 0 → 100644
View file @ e79b30ea
This diff is collapsed.
module3/exo1/analyse-syndrome-grippal.ipynb
View file @ e79b30ea
...@@ -9,14 +9,19 @@ ...@@ -9,14 +9,19 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 51,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"%matplotlib inline\n", "%matplotlib inline\n",
"import matplotlib.pyplot as plt\n", "import matplotlib.pyplot as plt\n",
"import pandas as pd\n", "import pandas as pd\n",
"import isoweek" "import isoweek\n",
"import hashlib\n",
"import requests\n",
"import os\n",
"import glob\n",
"from datetime import datetime"
] ]
}, },
{ {
...@@ -28,13 +33,36 @@ ...@@ -28,13 +33,36 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 67,
"metadata": { "metadata": {},
"collapsed": true "outputs": [],
}, "source": [
"data_url = \"http://www.sentiweb.fr/datasets/incidence-PAY-3.csv\"\n",
"local_file_pattern = \"-sentinelle-syndrome-grippal.csv\""
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Afin de verifier que la copie locale est identique au valeurs disponible en ligne à l'URL fournis, on procede avec : \n",
"1. Les données sont elles encore disponible à l'URL indiqué `data_url`\n",
"2. Le hash (sha2556) de ces données est-il identique a celle deja presente en local `local_file`\n",
" 1. Si oui -> pas d'action\n",
" 2. Si non -> télecharger les nouvelles datas, il faudrat aussi indiquer que le fichier a ete mis à jour et conserver le numero de version avec `%y%m%d-sentinelle-syndrome-grippal.csv`"
]
},
{
"cell_type": "code",
"execution_count": 39,
"metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"data_url = \"http://www.sentiweb.fr/datasets/incidence-PAY-3.csv\"" "# fonctions permettant de calculer le sha256\n",
"def computesha256(content): \n",
" sh256 = hashlib.sha256()\n",
" sh256.update(content)\n",
" return sh256.hexdigest()"
] ]
}, },
{ {
...@@ -61,12 +89,202 @@ ...@@ -61,12 +89,202 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 80,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Detected 1 file(s) with pattern -sentinelle-syndrome-grippal.csv\n",
"Using latest : 20251210-sentinelle-syndrome-grippal.csv\n",
"Checking data version :\n",
"Local file sentinelle-syndrome-grippal.csv is identical to provided the one at provided URL, no need to update data\n",
"<class 'pandas.core.frame.DataFrame'>\n",
"RangeIndex: 2144 entries, 0 to 2143\n",
"Data columns (total 10 columns):\n",
"week 2144 non-null int64\n",
"indicator 2144 non-null int64\n",
"inc 2144 non-null object\n",
"inc_low 2143 non-null float64\n",
"inc_up 2143 non-null float64\n",
"inc100 2144 non-null object\n",
"inc100_low 2143 non-null float64\n",
"inc100_up 2143 non-null float64\n",
"geo_insee 2144 non-null object\n",
"geo_name 2144 non-null object\n",
"dtypes: float64(4), int64(2), object(4)\n",
"memory usage: 167.6+ KB\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>week</th>\n",
" <th>indicator</th>\n",
" <th>inc</th>\n",
" <th>inc_low</th>\n",
" <th>inc_up</th>\n",
" <th>inc100</th>\n",
" <th>inc100_low</th>\n",
" <th>inc100_up</th>\n",
" <th>geo_insee</th>\n",
" <th>geo_name</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>202548</td>\n",
" <td>3</td>\n",
" <td>85628</td>\n",
" <td>75478.0</td>\n",
" <td>95778.0</td>\n",
" <td>128</td>\n",
" <td>113.0</td>\n",
" <td>143.0</td>\n",
" <td>FR</td>\n",
" <td>France</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>202547</td>\n",
" <td>3</td>\n",
" <td>59094</td>\n",
" <td>51658.0</td>\n",
" <td>66530.0</td>\n",
" <td>88</td>\n",
" <td>77.0</td>\n",
" <td>99.0</td>\n",
" <td>FR</td>\n",
" <td>France</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>202546</td>\n",
" <td>3</td>\n",
" <td>40657</td>\n",
" <td>34407.0</td>\n",
" <td>46907.0</td>\n",
" <td>61</td>\n",
" <td>52.0</td>\n",
" <td>70.0</td>\n",
" <td>FR</td>\n",
" <td>France</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>202545</td>\n",
" <td>3</td>\n",
" <td>47645</td>\n",
" <td>40423.0</td>\n",
" <td>54867.0</td>\n",
" <td>71</td>\n",
" <td>60.0</td>\n",
" <td>82.0</td>\n",
" <td>FR</td>\n",
" <td>France</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>202544</td>\n",
" <td>3</td>\n",
" <td>44837</td>\n",
" <td>38385.0</td>\n",
" <td>51289.0</td>\n",
" <td>67</td>\n",
" <td>57.0</td>\n",
" <td>77.0</td>\n",
" <td>FR</td>\n",
" <td>France</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" week indicator inc inc_low inc_up inc100 inc100_low inc100_up \\\n",
"0 202548 3 85628 75478.0 95778.0 128 113.0 143.0 \n",
"1 202547 3 59094 51658.0 66530.0 88 77.0 99.0 \n",
"2 202546 3 40657 34407.0 46907.0 61 52.0 70.0 \n",
"3 202545 3 47645 40423.0 54867.0 71 60.0 82.0 \n",
"4 202544 3 44837 38385.0 51289.0 67 57.0 77.0 \n",
"\n",
" geo_insee geo_name \n",
"0 FR France \n",
"1 FR France \n",
"2 FR France \n",
"3 FR France \n",
"4 FR France "
]
},
"execution_count": 80,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [ "source": [
"raw_data = pd.read_csv(data_url, skiprows=1)\n", "# checking if URL is still active \n",
"raw_data" "url_file = requests.get(data_url, timeout = 50)\n",
"assert url_file.raise_for_status() != 200, f'Provided URL is unreachable or returned {url_file.status_code}'\n",
"\n",
"# loading file from URL\n",
"url_content = url_file.content\n",
"\n",
"# compute sha256 \n",
"url_sha256 = computesha256(url_content)\n",
"\n",
"\n",
"# detect latest version of local data \n",
"files = glob.glob(f'*{local_file_pattern}')\n",
"\n",
"\n",
"if files == [] : \n",
" print(f'No local file detected using pattern {local_file_pattern}, ... Downloading up to date')\n",
" with open(f'{datetime.today().strftime(\"%Y%m%d\")}-{local_file_pattern}', 'wb') as f:\n",
" f.write(url_content)\n",
"else : \n",
" print(f'Detected {len(files)} file(s) with pattern {local_file_pattern}')\n",
" latest = max(files) \n",
" print(f'Using latest : {latest}')\n",
" with open(latest, 'rb') as f:\n",
" data= f.read()\n",
" local_sha256 = computesha256(data)\n",
"\n",
" print('Checking data version :')\n",
" if url_sha256 == local_sha256 : \n",
" print(f'Local file {local_file} is identical to provided the one at provided URL, no need to update data')\n",
" raw_data = local_file\n",
" \n",
" else : \n",
" print(f'The current file {local_file}, is not up to date compared to URL, ... Downloading the new one')\n",
" update = f'{datetime.today().strftime(\"%Y%m%d\")}-{local_file_pattern}'\n",
" with open(update, 'wb') as f:\n",
" f.write(url_content)\n",
" raw_data = update\n",
"\n",
"raw_data = pd.read_csv(raw_data, encoding = 'iso-8859-1', skiprows=1) \n",
"\n",
"raw_data.info()\n",
"raw_data.head()"
] ]
}, },
{ {
...@@ -364,7 +582,7 @@ ...@@ -364,7 +582,7 @@
"name": "python", "name": "python",
"nbconvert_exporter": "python", "nbconvert_exporter": "python",
"pygments_lexer": "ipython3", "pygments_lexer": "ipython3",
"version": "3.6.1" "version": "3.6.4"
} }
}, },
"nbformat": 4, "nbformat": 4,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment