Skip to content

M
mooc-rr
Commit e79b30ea authored by fcf61da24638f93d44cd057b11a6abdc's avatar fcf61da24638f93d44cd057b11a6abdc
Browse files
Options

initial commit, added sha256 and automatic versionning of file from url

parent 8c63b547
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 2376 additions and 12 deletions
module3/exo1/20251210-sentinelle-syndrome-grippal.csv 0 → 100644
View file @ e79b30ea
This diff is collapsed.
module3/exo1/analyse-syndrome-grippal.ipynb
View file @ e79b30ea
......@@ -9,14 +9,19 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 51,
"metadata": {},
"outputs": [],
"source": [
"%matplotlib inline\n",
"import matplotlib.pyplot as plt\n",
"import pandas as pd\n",
"import isoweek"
"import isoweek\n",
"import hashlib\n",
"import requests\n",
"import os\n",
"import glob\n",
"from datetime import datetime"
]
},
{
......@@ -28,13 +33,36 @@
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"execution_count": 67,
"metadata": {},
"outputs": [],
"source": [
"data_url = \"http://www.sentiweb.fr/datasets/incidence-PAY-3.csv\"\n",
"local_file_pattern = \"-sentinelle-syndrome-grippal.csv\""
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Afin de verifier que la copie locale est identique au valeurs disponible en ligne à l'URL fournis, on procede avec : \n",
"1. Les données sont elles encore disponible à l'URL indiqué `data_url`\n",
"2. Le hash (sha2556) de ces données est-il identique a celle deja presente en local `local_file`\n",
" 1. Si oui -> pas d'action\n",
" 2. Si non -> télecharger les nouvelles datas, il faudrat aussi indiquer que le fichier a ete mis à jour et conserver le numero de version avec `%y%m%d-sentinelle-syndrome-grippal.csv`"
]
},
{
"cell_type": "code",
"execution_count": 39,
"metadata": {},
"outputs": [],
"source": [
"data_url = \"http://www.sentiweb.fr/datasets/incidence-PAY-3.csv\""
"# fonctions permettant de calculer le sha256\n",
"def computesha256(content): \n",
" sh256 = hashlib.sha256()\n",
" sh256.update(content)\n",
" return sh256.hexdigest()"
]
},
{
......@@ -61,12 +89,202 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 80,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Detected 1 file(s) with pattern -sentinelle-syndrome-grippal.csv\n",
"Using latest : 20251210-sentinelle-syndrome-grippal.csv\n",
"Checking data version :\n",
"Local file sentinelle-syndrome-grippal.csv is identical to provided the one at provided URL, no need to update data\n",
"<class 'pandas.core.frame.DataFrame'>\n",
"RangeIndex: 2144 entries, 0 to 2143\n",
"Data columns (total 10 columns):\n",
"week 2144 non-null int64\n",
"indicator 2144 non-null int64\n",
"inc 2144 non-null object\n",
"inc_low 2143 non-null float64\n",
"inc_up 2143 non-null float64\n",
"inc100 2144 non-null object\n",
"inc100_low 2143 non-null float64\n",
"inc100_up 2143 non-null float64\n",
"geo_insee 2144 non-null object\n",
"geo_name 2144 non-null object\n",
"dtypes: float64(4), int64(2), object(4)\n",
"memory usage: 167.6+ KB\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>week</th>\n",
" <th>indicator</th>\n",
" <th>inc</th>\n",
" <th>inc_low</th>\n",
" <th>inc_up</th>\n",
" <th>inc100</th>\n",
" <th>inc100_low</th>\n",
" <th>inc100_up</th>\n",
" <th>geo_insee</th>\n",
" <th>geo_name</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>202548</td>\n",
" <td>3</td>\n",
" <td>85628</td>\n",
" <td>75478.0</td>\n",
" <td>95778.0</td>\n",
" <td>128</td>\n",
" <td>113.0</td>\n",
" <td>143.0</td>\n",
" <td>FR</td>\n",
" <td>France</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>202547</td>\n",
" <td>3</td>\n",
" <td>59094</td>\n",
" <td>51658.0</td>\n",
" <td>66530.0</td>\n",
" <td>88</td>\n",
" <td>77.0</td>\n",
" <td>99.0</td>\n",
" <td>FR</td>\n",
" <td>France</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>202546</td>\n",
" <td>3</td>\n",
" <td>40657</td>\n",
" <td>34407.0</td>\n",
" <td>46907.0</td>\n",
" <td>61</td>\n",
" <td>52.0</td>\n",
" <td>70.0</td>\n",
" <td>FR</td>\n",
" <td>France</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>202545</td>\n",
" <td>3</td>\n",
" <td>47645</td>\n",
" <td>40423.0</td>\n",
" <td>54867.0</td>\n",
" <td>71</td>\n",
" <td>60.0</td>\n",
" <td>82.0</td>\n",
" <td>FR</td>\n",
" <td>France</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>202544</td>\n",
" <td>3</td>\n",
" <td>44837</td>\n",
" <td>38385.0</td>\n",
" <td>51289.0</td>\n",
" <td>67</td>\n",
" <td>57.0</td>\n",
" <td>77.0</td>\n",
" <td>FR</td>\n",
" <td>France</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" week indicator inc inc_low inc_up inc100 inc100_low inc100_up \\\n",
"0 202548 3 85628 75478.0 95778.0 128 113.0 143.0 \n",
"1 202547 3 59094 51658.0 66530.0 88 77.0 99.0 \n",
"2 202546 3 40657 34407.0 46907.0 61 52.0 70.0 \n",
"3 202545 3 47645 40423.0 54867.0 71 60.0 82.0 \n",
"4 202544 3 44837 38385.0 51289.0 67 57.0 77.0 \n",
"\n",
" geo_insee geo_name \n",
"0 FR France \n",
"1 FR France \n",
"2 FR France \n",
"3 FR France \n",
"4 FR France "
]
},
"execution_count": 80,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"raw_data = pd.read_csv(data_url, skiprows=1)\n",
"raw_data"
"# checking if URL is still active \n",
"url_file = requests.get(data_url, timeout = 50)\n",
"assert url_file.raise_for_status() != 200, f'Provided URL is unreachable or returned {url_file.status_code}'\n",
"\n",
"# loading file from URL\n",
"url_content = url_file.content\n",
"\n",
"# compute sha256 \n",
"url_sha256 = computesha256(url_content)\n",
"\n",
"\n",
"# detect latest version of local data \n",
"files = glob.glob(f'*{local_file_pattern}')\n",
"\n",
"\n",
"if files == [] : \n",
" print(f'No local file detected using pattern {local_file_pattern}, ... Downloading up to date')\n",
" with open(f'{datetime.today().strftime(\"%Y%m%d\")}-{local_file_pattern}', 'wb') as f:\n",
" f.write(url_content)\n",
"else : \n",
" print(f'Detected {len(files)} file(s) with pattern {local_file_pattern}')\n",
" latest = max(files) \n",
" print(f'Using latest : {latest}')\n",
" with open(latest, 'rb') as f:\n",
" data= f.read()\n",
" local_sha256 = computesha256(data)\n",
"\n",
" print('Checking data version :')\n",
" if url_sha256 == local_sha256 : \n",
" print(f'Local file {local_file} is identical to provided the one at provided URL, no need to update data')\n",
" raw_data = local_file\n",
" \n",
" else : \n",
" print(f'The current file {local_file}, is not up to date compared to URL, ... Downloading the new one')\n",
" update = f'{datetime.today().strftime(\"%Y%m%d\")}-{local_file_pattern}'\n",
" with open(update, 'wb') as f:\n",
" f.write(url_content)\n",
" raw_data = update\n",
"\n",
"raw_data = pd.read_csv(raw_data, encoding = 'iso-8859-1', skiprows=1) \n",
"\n",
"raw_data.info()\n",
"raw_data.head()"
]
},
{
......@@ -364,7 +582,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.1"
"version": "3.6.4"
}
},
"nbformat": 4,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment