mozilla
/
UCOSP-winter-2018_TrackingTechnologies
зеркало из
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Merge pull request #4 from Alexander1994/cookie_data_cleanup 

Cleaned up Extract cookie notebook and added scraper to pull cookie info with a simple cache
This commit is contained in:
mlopatka 2018-04-18 13:59:07 +02:00 коммит произвёл GitHub
Родитель 7afbf81970 85946e745d
Коммит 9e23f8582b
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
4 изменённых файлов: 14987 добавлений и 883 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

7470
analyses/Extract Cookies & Storage Data.ipynb Normal file
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

7470
analyses/Extract Cookies & Storage Data.md Normal file
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

883
analyses/extract_cookie.ipynb
Просмотреть файл

@ -1,883 +0,0 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"import pandas as pd\n",
"import json\n",
"import sys\n",
"sys.path.append('..')\n",
"from utils.load_data_util import load_random_data\n",
"\n",
"result = load_random_data(10, seed=42)"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"scrolled": true
},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>key</th>\n",
" <th>value</th>\n",
" <th>script_url</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>RNLBSERVERID</td>\n",
" <td>ded6726</td>\n",
" <td>https://syndication.exosrv.com/splash.php?idzo...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>_ga</td>\n",
" <td>GA1.2.692713596.1513387628</td>\n",
" <td>https://syndication.exosrv.com/splash.php?idzo...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>_gid</td>\n",
" <td>GA1.2.1540566351.1513387628</td>\n",
" <td>https://syndication.exosrv.com/splash.php?idzo...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>_gat</td>\n",
" <td>1</td>\n",
" <td>https://syndication.exosrv.com/splash.php?idzo...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>splash_i</td>\n",
" <td>false</td>\n",
" <td>https://syndication.exosrv.com/splash.php?idzo...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>expires</td>\n",
" <td>Sun, 16 Dec 2018 01:27:12 GMT</td>\n",
" <td>https://syndication.exosrv.com/splash.php?idzo...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>path</td>\n",
" <td>/</td>\n",
" <td>https://syndication.exosrv.com/splash.php?idzo...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>Adshow</td>\n",
" <td>1</td>\n",
" <td>http://ydjs.zol.com.cn/m.html?mediaid=c8de8cfb...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>z_pro_city</td>\n",
" <td>s_provice%3Dmixiegenzhou%26s_city%3Dnull</td>\n",
" <td>http://ydjs.zol.com.cn/m.html?mediaid=c8de8cfb...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>userProvinceId</td>\n",
" <td>1</td>\n",
" <td>http://ydjs.zol.com.cn/m.html?mediaid=c8de8cfb...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>10</th>\n",
" <td>userCityId</td>\n",
" <td>0</td>\n",
" <td>http://ydjs.zol.com.cn/m.html?mediaid=c8de8cfb...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>11</th>\n",
" <td>userCountyId</td>\n",
" <td>0</td>\n",
" <td>http://ydjs.zol.com.cn/m.html?mediaid=c8de8cfb...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>12</th>\n",
" <td>userLocationId</td>\n",
" <td>1</td>\n",
" <td>http://ydjs.zol.com.cn/m.html?mediaid=c8de8cfb...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>13</th>\n",
" <td>ip_ck</td>\n",
" <td>4c+H5PP1j7QuNjg2MzkyLjE1MTM0MTU0NzQ%3D</td>\n",
" <td>http://ydjs.zol.com.cn/m.html?mediaid=c8de8cfb...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>14</th>\n",
" <td>lv</td>\n",
" <td>1513415476</td>\n",
" <td>http://ydjs.zol.com.cn/m.html?mediaid=c8de8cfb...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>15</th>\n",
" <td>vn</td>\n",
" <td>1</td>\n",
" <td>http://ydjs.zol.com.cn/m.html?mediaid=c8de8cfb...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16</th>\n",
" <td>Adshow</td>\n",
" <td>1</td>\n",
" <td>http://ydjs.zol.com.cn/m.html?mediaid=c8de8cfb...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>17</th>\n",
" <td>z_pro_city</td>\n",
" <td>s_provice%3Dmixiegenzhou%26s_city%3Dnull</td>\n",
" <td>http://ydjs.zol.com.cn/m.html?mediaid=c8de8cfb...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>18</th>\n",
" <td>userProvinceId</td>\n",
" <td>1</td>\n",
" <td>http://ydjs.zol.com.cn/m.html?mediaid=c8de8cfb...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>19</th>\n",
" <td>userCityId</td>\n",
" <td>0</td>\n",
" <td>http://ydjs.zol.com.cn/m.html?mediaid=c8de8cfb...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>20</th>\n",
" <td>userCountyId</td>\n",
" <td>0</td>\n",
" <td>http://ydjs.zol.com.cn/m.html?mediaid=c8de8cfb...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>21</th>\n",
" <td>userLocationId</td>\n",
" <td>1</td>\n",
" <td>http://ydjs.zol.com.cn/m.html?mediaid=c8de8cfb...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>22</th>\n",
" <td>ip_ck</td>\n",
" <td>4c+H5PP1j7QuNjg2MzkyLjE1MTM0MTU0NzQ%3D</td>\n",
" <td>http://ydjs.zol.com.cn/m.html?mediaid=c8de8cfb...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>23</th>\n",
" <td>lv</td>\n",
" <td>1513415476</td>\n",
" <td>http://ydjs.zol.com.cn/m.html?mediaid=c8de8cfb...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>24</th>\n",
" <td>vn</td>\n",
" <td>1</td>\n",
" <td>http://ydjs.zol.com.cn/m.html?mediaid=c8de8cfb...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>25</th>\n",
" <td>POSMEDIAID</td>\n",
" <td>c8de8cfb85858ad6c30636190806b8fc9b43af469b42ff...</td>\n",
" <td>http://ydjs.zol.com.cn/m.html?mediaid=c8de8cfb...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>26</th>\n",
" <td>path</td>\n",
" <td>/</td>\n",
" <td>http://ydjs.zol.com.cn/m.html?mediaid=c8de8cfb...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>27</th>\n",
" <td>expires</td>\n",
" <td>Thu, 30 Nov 2090 18:22:56 GMT</td>\n",
" <td>http://ydjs.zol.com.cn/m.html?mediaid=c8de8cfb...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>28</th>\n",
" <td>domain</td>\n",
" <td>.ydjs.zol.com.cn</td>\n",
" <td>http://ydjs.zol.com.cn/m.html?mediaid=c8de8cfb...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>29</th>\n",
" <td>sd</td>\n",
" <td>1</td>\n",
" <td>https://content.adriver.ru/banners/0002186/000...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4822</th>\n",
" <td>__utmt</td>\n",
" <td>1</td>\n",
" <td>http://stats.g.doubleclick.net/dc.js</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4823</th>\n",
" <td>__utma</td>\n",
" <td>219845656.2053000654.1513465816.1513465816.151...</td>\n",
" <td>http://stats.g.doubleclick.net/dc.js</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4824</th>\n",
" <td>__utmb</td>\n",
" <td>219845656.1.10.1513465816</td>\n",
" <td>http://stats.g.doubleclick.net/dc.js</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4825</th>\n",
" <td>__utmc</td>\n",
" <td>219845656</td>\n",
" <td>http://stats.g.doubleclick.net/dc.js</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4826</th>\n",
" <td>__utmz</td>\n",
" <td>219845656.1513465816.1.1.utmcsr=(direct)|utmcc...</td>\n",
" <td>http://stats.g.doubleclick.net/dc.js</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4827</th>\n",
" <td>__utmt</td>\n",
" <td>1</td>\n",
" <td>http://stats.g.doubleclick.net/dc.js</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4828</th>\n",
" <td>__utma</td>\n",
" <td>219845656.2053000654.1513465816.1513465816.151...</td>\n",
" <td>http://stats.g.doubleclick.net/dc.js</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4829</th>\n",
" <td>__utmb</td>\n",
" <td>219845656.1.10.1513465816</td>\n",
" <td>http://stats.g.doubleclick.net/dc.js</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4830</th>\n",
" <td>__utmc</td>\n",
" <td>219845656</td>\n",
" <td>http://stats.g.doubleclick.net/dc.js</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4831</th>\n",
" <td>__utmz</td>\n",
" <td>219845656.1513465816.1.1.utmcsr=(direct)|utmcc...</td>\n",
" <td>http://stats.g.doubleclick.net/dc.js</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4832</th>\n",
" <td>__utmt</td>\n",
" <td>1</td>\n",
" <td>http://stats.g.doubleclick.net/dc.js</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4833</th>\n",
" <td>__utma</td>\n",
" <td>219845656.2053000654.1513465816.1513465816.151...</td>\n",
" <td>http://stats.g.doubleclick.net/dc.js</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4834</th>\n",
" <td>path</td>\n",
" <td>/</td>\n",
" <td>http://stats.g.doubleclick.net/dc.js</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4835</th>\n",
" <td>expires</td>\n",
" <td>Mon, 16 Dec 2019 23:10:15 GMT</td>\n",
" <td>http://stats.g.doubleclick.net/dc.js</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4836</th>\n",
" <td>domain</td>\n",
" <td>pepper.pr.co</td>\n",
" <td>http://stats.g.doubleclick.net/dc.js</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4837</th>\n",
" <td>__utmb</td>\n",
" <td>219845656.2.9.1513465816</td>\n",
" <td>http://stats.g.doubleclick.net/dc.js</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4838</th>\n",
" <td>path</td>\n",
" <td>/</td>\n",
" <td>http://stats.g.doubleclick.net/dc.js</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4839</th>\n",
" <td>expires</td>\n",
" <td>Sat, 16 Dec 2017 23:40:15 GMT</td>\n",
" <td>http://stats.g.doubleclick.net/dc.js</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4840</th>\n",
" <td>domain</td>\n",
" <td>pepper.pr.co</td>\n",
" <td>http://stats.g.doubleclick.net/dc.js</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4841</th>\n",
" <td>__utmc</td>\n",
" <td>219845656</td>\n",
" <td>http://stats.g.doubleclick.net/dc.js</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4842</th>\n",
" <td>path</td>\n",
" <td>/</td>\n",
" <td>http://stats.g.doubleclick.net/dc.js</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4843</th>\n",
" <td>domain</td>\n",
" <td>pepper.pr.co</td>\n",
" <td>http://stats.g.doubleclick.net/dc.js</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4844</th>\n",
" <td>__utmz</td>\n",
" <td>219845656.1513465816.1.1.utmcsr=(direct)|utmcc...</td>\n",
" <td>http://stats.g.doubleclick.net/dc.js</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4845</th>\n",
" <td>path</td>\n",
" <td>/</td>\n",
" <td>http://stats.g.doubleclick.net/dc.js</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4846</th>\n",
" <td>expires</td>\n",
" <td>Sun, 17 Jun 2018 11:10:15 GMT</td>\n",
" <td>http://stats.g.doubleclick.net/dc.js</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4847</th>\n",
" <td>domain</td>\n",
" <td>pepper.pr.co</td>\n",
" <td>http://stats.g.doubleclick.net/dc.js</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4848</th>\n",
" <td>__utmv</td>\n",
" <td></td>\n",
" <td>http://stats.g.doubleclick.net/dc.js</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4849</th>\n",
" <td>path</td>\n",
" <td>/</td>\n",
" <td>http://stats.g.doubleclick.net/dc.js</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4850</th>\n",
" <td>expires</td>\n",
" <td>Sat, 16 Dec 2017 23:10:15 GMT</td>\n",
" <td>http://stats.g.doubleclick.net/dc.js</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4851</th>\n",
" <td>domain</td>\n",
" <td>pepper.pr.co</td>\n",
" <td>http://stats.g.doubleclick.net/dc.js</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>4852 rows × 3 columns</p>\n",
"</div>"
],
"text/plain": [
" key value script_url\n",
"0 RNLBSERVERID ded6726 https://syndication.exosrv.com/splash.php?idzo...\n",
"1 _ga GA1.2.692713596.1513387628 https://syndication.exosrv.com/splash.php?idzo...\n",
"2 _gid GA1.2.1540566351.1513387628 https://syndication.exosrv.com/splash.php?idzo...\n",
"3 _gat 1 https://syndication.exosrv.com/splash.php?idzo...\n",
"4 splash_i false https://syndication.exosrv.com/splash.php?idzo...\n",
"5 expires Sun, 16 Dec 2018 01:27:12 GMT https://syndication.exosrv.com/splash.php?idzo...\n",
"6 path / https://syndication.exosrv.com/splash.php?idzo...\n",
"7 Adshow 1 http://ydjs.zol.com.cn/m.html?mediaid=c8de8cfb...\n",
"8 z_pro_city s_provice%3Dmixiegenzhou%26s_city%3Dnull http://ydjs.zol.com.cn/m.html?mediaid=c8de8cfb...\n",
"9 userProvinceId 1 http://ydjs.zol.com.cn/m.html?mediaid=c8de8cfb...\n",
"10 userCityId 0 http://ydjs.zol.com.cn/m.html?mediaid=c8de8cfb...\n",
"11 userCountyId 0 http://ydjs.zol.com.cn/m.html?mediaid=c8de8cfb...\n",
"12 userLocationId 1 http://ydjs.zol.com.cn/m.html?mediaid=c8de8cfb...\n",
"13 ip_ck 4c+H5PP1j7QuNjg2MzkyLjE1MTM0MTU0NzQ%3D http://ydjs.zol.com.cn/m.html?mediaid=c8de8cfb...\n",
"14 lv 1513415476 http://ydjs.zol.com.cn/m.html?mediaid=c8de8cfb...\n",
"15 vn 1 http://ydjs.zol.com.cn/m.html?mediaid=c8de8cfb...\n",
"16 Adshow 1 http://ydjs.zol.com.cn/m.html?mediaid=c8de8cfb...\n",
"17 z_pro_city s_provice%3Dmixiegenzhou%26s_city%3Dnull http://ydjs.zol.com.cn/m.html?mediaid=c8de8cfb...\n",
"18 userProvinceId 1 http://ydjs.zol.com.cn/m.html?mediaid=c8de8cfb...\n",
"19 userCityId 0 http://ydjs.zol.com.cn/m.html?mediaid=c8de8cfb...\n",
"20 userCountyId 0 http://ydjs.zol.com.cn/m.html?mediaid=c8de8cfb...\n",
"21 userLocationId 1 http://ydjs.zol.com.cn/m.html?mediaid=c8de8cfb...\n",
"22 ip_ck 4c+H5PP1j7QuNjg2MzkyLjE1MTM0MTU0NzQ%3D http://ydjs.zol.com.cn/m.html?mediaid=c8de8cfb...\n",
"23 lv 1513415476 http://ydjs.zol.com.cn/m.html?mediaid=c8de8cfb...\n",
"24 vn 1 http://ydjs.zol.com.cn/m.html?mediaid=c8de8cfb...\n",
"25 POSMEDIAID c8de8cfb85858ad6c30636190806b8fc9b43af469b42ff... http://ydjs.zol.com.cn/m.html?mediaid=c8de8cfb...\n",
"26 path / http://ydjs.zol.com.cn/m.html?mediaid=c8de8cfb...\n",
"27 expires Thu, 30 Nov 2090 18:22:56 GMT http://ydjs.zol.com.cn/m.html?mediaid=c8de8cfb...\n",
"28 domain .ydjs.zol.com.cn http://ydjs.zol.com.cn/m.html?mediaid=c8de8cfb...\n",
"29 sd 1 https://content.adriver.ru/banners/0002186/000...\n",
"... ... ... ...\n",
"4822 __utmt 1 http://stats.g.doubleclick.net/dc.js\n",
"4823 __utma 219845656.2053000654.1513465816.1513465816.151... http://stats.g.doubleclick.net/dc.js\n",
"4824 __utmb 219845656.1.10.1513465816 http://stats.g.doubleclick.net/dc.js\n",
"4825 __utmc 219845656 http://stats.g.doubleclick.net/dc.js\n",
"4826 __utmz 219845656.1513465816.1.1.utmcsr=(direct)|utmcc... http://stats.g.doubleclick.net/dc.js\n",
"4827 __utmt 1 http://stats.g.doubleclick.net/dc.js\n",
"4828 __utma 219845656.2053000654.1513465816.1513465816.151... http://stats.g.doubleclick.net/dc.js\n",
"4829 __utmb 219845656.1.10.1513465816 http://stats.g.doubleclick.net/dc.js\n",
"4830 __utmc 219845656 http://stats.g.doubleclick.net/dc.js\n",
"4831 __utmz 219845656.1513465816.1.1.utmcsr=(direct)|utmcc... http://stats.g.doubleclick.net/dc.js\n",
"4832 __utmt 1 http://stats.g.doubleclick.net/dc.js\n",
"4833 __utma 219845656.2053000654.1513465816.1513465816.151... http://stats.g.doubleclick.net/dc.js\n",
"4834 path / http://stats.g.doubleclick.net/dc.js\n",
"4835 expires Mon, 16 Dec 2019 23:10:15 GMT http://stats.g.doubleclick.net/dc.js\n",
"4836 domain pepper.pr.co http://stats.g.doubleclick.net/dc.js\n",
"4837 __utmb 219845656.2.9.1513465816 http://stats.g.doubleclick.net/dc.js\n",
"4838 path / http://stats.g.doubleclick.net/dc.js\n",
"4839 expires Sat, 16 Dec 2017 23:40:15 GMT http://stats.g.doubleclick.net/dc.js\n",
"4840 domain pepper.pr.co http://stats.g.doubleclick.net/dc.js\n",
"4841 __utmc 219845656 http://stats.g.doubleclick.net/dc.js\n",
"4842 path / http://stats.g.doubleclick.net/dc.js\n",
"4843 domain pepper.pr.co http://stats.g.doubleclick.net/dc.js\n",
"4844 __utmz 219845656.1513465816.1.1.utmcsr=(direct)|utmcc... http://stats.g.doubleclick.net/dc.js\n",
"4845 path / http://stats.g.doubleclick.net/dc.js\n",
"4846 expires Sun, 17 Jun 2018 11:10:15 GMT http://stats.g.doubleclick.net/dc.js\n",
"4847 domain pepper.pr.co http://stats.g.doubleclick.net/dc.js\n",
"4848 __utmv http://stats.g.doubleclick.net/dc.js\n",
"4849 path / http://stats.g.doubleclick.net/dc.js\n",
"4850 expires Sat, 16 Dec 2017 23:10:15 GMT http://stats.g.doubleclick.net/dc.js\n",
"4851 domain pepper.pr.co http://stats.g.doubleclick.net/dc.js\n",
"\n",
"[4852 rows x 3 columns]"
]
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"result_cookie = result.loc[(result['symbol'] == \"window.document.cookie\") & (result['value'].str.contains(\"=\"))]\n",
"cookiedf = pd.DataFrame(columns=['key', 'value', 'script_url'])\n",
"for i, row in result_cookie.iterrows():\n",
" for kv in row['value'].split(\";\"):\n",
" if \"=\" in kv:\n",
" keyValueArr = kv.strip().split(\"=\", maxsplit=1)\n",
" cookiedf = cookiedf.append({'key':keyValueArr[0], 'value':keyValueArr[1], 'script_url':row['script_url']}, ignore_index=True) \n",
"\n",
"cookiedf\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[{'json': {'i18nextLng': 'en-US'},\n",
" 'script_url': 'https://syndication.exosrv.com/splash.php?idzone=1931806&type=3&sub=1'},\n",
" {'json': {},\n",
" 'script_url': 'http://d1m6l9dfulcyw7.cloudfront.net/uxa/a82093cd9cdbf.js'},\n",
" {'json': {'CSLocalStorageTest': 'CSLocalStorageTest'},\n",
" 'script_url': 'http://d1m6l9dfulcyw7.cloudfront.net/uxa/a82093cd9cdbf.js'},\n",
" {'json': {},\n",
" 'script_url': 'http://d1m6l9dfulcyw7.cloudfront.net/uxa/a82093cd9cdbf.js'},\n",
" {'json': {},\n",
" 'script_url': 'http://d1m6l9dfulcyw7.cloudfront.net/uxa/a82093cd9cdbf.js'},\n",
" {'json': {'CSStorageData:timeStamp': '1513385259071'},\n",
" 'script_url': 'http://d1m6l9dfulcyw7.cloudfront.net/uxa/a82093cd9cdbf.js'},\n",
" {'json': {'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SCROLL_RATE': '0',\n",
" 'CSStorageData:timeStamp': '1513385259071'},\n",
" 'script_url': 'https://js.datadome.co/tags.js'},\n",
" {'json': {'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SCROLL_RATE': '0',\n",
" 'CSStorageData:timeStamp': '1513385259071'},\n",
" 'script_url': 'http://d1m6l9dfulcyw7.cloudfront.net/uxa/a82093cd9cdbf.js'},\n",
" {'json': {'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SCROLL_RATE': '7',\n",
" 'CSStorageData:timeStamp': '1513385259071'},\n",
" 'script_url': 'http://d1m6l9dfulcyw7.cloudfront.net/uxa/a82093cd9cdbf.js'},\n",
" {'json': {'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SCROLL_RATE': '7',\n",
" 'CSStorageData:timeStamp': '1513385259071'},\n",
" 'script_url': 'http://d1m6l9dfulcyw7.cloudfront.net/uxa/a82093cd9cdbf.js'},\n",
" {'json': {'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:PROJECT_ID': '269',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SCROLL_RATE': '7',\n",
" 'CSStorageData:timeStamp': '1513385259071'},\n",
" 'script_url': 'http://d1m6l9dfulcyw7.cloudfront.net/uxa/a82093cd9cdbf.js'},\n",
" {'json': {'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:CACHE_BUSTING': '\"263385\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:PROJECT_ID': '269',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SCROLL_RATE': '7',\n",
" 'CSStorageData:timeStamp': '1513385259071'},\n",
" 'script_url': 'http://d1m6l9dfulcyw7.cloudfront.net/uxa/a82093cd9cdbf.js'},\n",
" {'json': {'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:CACHE_BUSTING': '\"263385\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:PROJECT_ID': '269',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SCROLL_RATE': '7',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:USER_ID': '\"d8151d4d-1a7c-a810-c2dd-5b1cc3a0f286\"',\n",
" 'CSStorageData:timeStamp': '1513385259071'},\n",
" 'script_url': 'http://d1m6l9dfulcyw7.cloudfront.net/uxa/a82093cd9cdbf.js'},\n",
" {'json': {'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:CACHE_BUSTING': '\"263385\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:PROJECT_ID': '269',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SCROLL_RATE': '7',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SESSION_NUMBER': '\"1\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:USER_ID': '\"d8151d4d-1a7c-a810-c2dd-5b1cc3a0f286\"',\n",
" 'CSStorageData:timeStamp': '1513385259071'},\n",
" 'script_url': 'http://d1m6l9dfulcyw7.cloudfront.net/uxa/a82093cd9cdbf.js'},\n",
" {'json': {'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:CACHE_BUSTING': '\"263385\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:LAST_VISIT': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:PROJECT_ID': '269',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SCROLL_RATE': '7',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SESSION_NUMBER': '\"1\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:USER_ID': '\"d8151d4d-1a7c-a810-c2dd-5b1cc3a0f286\"',\n",
" 'CSStorageData:timeStamp': '1513385259071'},\n",
" 'script_url': 'http://d1m6l9dfulcyw7.cloudfront.net/uxa/a82093cd9cdbf.js'},\n",
" {'json': {'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:CACHE_BUSTING': '\"263385\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:LAST_HIT_DATE': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:LAST_VISIT': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:PROJECT_ID': '269',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SCROLL_RATE': '7',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SESSION_NUMBER': '\"1\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:USER_ID': '\"d8151d4d-1a7c-a810-c2dd-5b1cc3a0f286\"',\n",
" 'CSStorageData:timeStamp': '1513385259071'},\n",
" 'script_url': 'http://d1m6l9dfulcyw7.cloudfront.net/uxa/a82093cd9cdbf.js'},\n",
" {'json': {'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:CACHE_BUSTING': '\"263385\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:HIT_DATE': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:LAST_HIT_DATE': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:LAST_VISIT': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:PROJECT_ID': '269',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SCROLL_RATE': '7',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SESSION_NUMBER': '\"1\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:USER_ID': '\"d8151d4d-1a7c-a810-c2dd-5b1cc3a0f286\"',\n",
" 'CSStorageData:timeStamp': '1513385259071'},\n",
" 'script_url': 'http://d1m6l9dfulcyw7.cloudfront.net/uxa/a82093cd9cdbf.js'},\n",
" {'json': {'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:CACHE_BUSTING': '\"263385\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:HIT_DATE': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:LAST_HIT_DATE': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:LAST_VISIT': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:PAGE_NUMBER': '\"1\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:PROJECT_ID': '269',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SCROLL_RATE': '7',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SESSION_NUMBER': '\"1\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:USER_ID': '\"d8151d4d-1a7c-a810-c2dd-5b1cc3a0f286\"',\n",
" 'CSStorageData:timeStamp': '1513385259071'},\n",
" 'script_url': 'http://d1m6l9dfulcyw7.cloudfront.net/uxa/a82093cd9cdbf.js'},\n",
" {'json': {'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:CACHE_BUSTING': '\"263385\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:HIT_DATE': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:LAST_HIT_DATE': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:LAST_VISIT': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:PAGE_NUMBER': '\"1\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:PROJECT_ID': '269',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:RECORDING': '\"0\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SCROLL_RATE': '7',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SESSION_NUMBER': '\"1\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:USER_ID': '\"d8151d4d-1a7c-a810-c2dd-5b1cc3a0f286\"',\n",
" 'CSStorageData:timeStamp': '1513385259071'},\n",
" 'script_url': 'http://d1m6l9dfulcyw7.cloudfront.net/uxa/a82093cd9cdbf.js'},\n",
" {'json': {'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:CACHE_BUSTING': '\"263385\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:HIT_DATE': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:LAST_HIT_DATE': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:LAST_VISIT': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:PAGE_NUMBER': '\"1\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:PROJECT_ID': '269',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:RECORDING': '\"0\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SCROLL_RATE': '7',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SESSION_NUMBER': '\"1\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:USER_ID': '\"d8151d4d-1a7c-a810-c2dd-5b1cc3a0f286\"',\n",
" 'CSStorageData:stores:412876:eventsStore:queues:events:0': '[0,3485,1366,697]',\n",
" 'CSStorageData:timeStamp': '1513385259071'},\n",
" 'script_url': 'http://static.criteo.net/js/ld/ld.js'},\n",
" {'json': {'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:CACHE_BUSTING': '\"263385\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:HIT_DATE': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:LAST_HIT_DATE': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:LAST_VISIT': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:PAGE_NUMBER': '\"1\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:PROJECT_ID': '269',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:RECORDING': '\"0\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SCROLL_RATE': '7',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SESSION_NUMBER': '\"1\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:USER_ID': '\"d8151d4d-1a7c-a810-c2dd-5b1cc3a0f286\"',\n",
" 'CSStorageData:stores:412876:eventsStore:queues:events:0': '[0,3485,1366,697]',\n",
" 'CSStorageData:timeStamp': '1513385259071'},\n",
" 'script_url': 'http://static.criteo.net/js/ld/ld.js'},\n",
" {'json': {'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:CACHE_BUSTING': '\"263385\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:HIT_DATE': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:LAST_HIT_DATE': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:LAST_VISIT': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:PAGE_NUMBER': '\"1\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:PROJECT_ID': '269',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:RECORDING': '\"0\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SCROLL_RATE': '7',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SESSION_NUMBER': '\"1\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:USER_ID': '\"d8151d4d-1a7c-a810-c2dd-5b1cc3a0f286\"',\n",
" 'CSStorageData:stores:412876:eventsStore:queues:events:0': '[0,3485,1366,697]',\n",
" 'CSStorageData:timeStamp': '1513385259071'},\n",
" 'script_url': 'http://static.criteo.net/js/ld/ld.js'},\n",
" {'json': {'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:CACHE_BUSTING': '\"263385\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:HIT_DATE': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:LAST_HIT_DATE': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:LAST_VISIT': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:PAGE_NUMBER': '\"1\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:PROJECT_ID': '269',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:RECORDING': '\"0\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SCROLL_RATE': '7',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SESSION_NUMBER': '\"1\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:USER_ID': '\"d8151d4d-1a7c-a810-c2dd-5b1cc3a0f286\"',\n",
" 'CSStorageData:stores:412876:eventsStore:queues:events:0': '[0,3485,1366,697]',\n",
" 'CSStorageData:timeStamp': '1513385259071'},\n",
" 'script_url': 'http://static.criteo.net/js/ld/ld.js'},\n",
" {'json': {'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:CACHE_BUSTING': '\"263385\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:HIT_DATE': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:LAST_HIT_DATE': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:LAST_VISIT': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:PAGE_NUMBER': '\"1\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:PROJECT_ID': '269',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:RECORDING': '\"0\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SCROLL_RATE': '7',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SESSION_NUMBER': '\"1\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:USER_ID': '\"d8151d4d-1a7c-a810-c2dd-5b1cc3a0f286\"',\n",
" 'CSStorageData:stores:412876:eventsStore:queues:events:0': '[0,3485,1366,697]',\n",
" 'CSStorageData:timeStamp': '1513385259071'},\n",
" 'script_url': 'http://static.criteo.net/js/ld/ld.js'},\n",
" {'json': {'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:CACHE_BUSTING': '\"263385\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:HIT_DATE': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:LAST_HIT_DATE': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:LAST_VISIT': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:PAGE_NUMBER': '\"1\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:PROJECT_ID': '269',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:RECORDING': '\"0\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SCROLL_RATE': '7',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SESSION_NUMBER': '\"1\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:USER_ID': '\"d8151d4d-1a7c-a810-c2dd-5b1cc3a0f286\"',\n",
" 'CSStorageData:stores:412876:eventsStore:queues:events:0': '[0,3485,1366,697]',\n",
" 'CSStorageData:timeStamp': '1513385259071'},\n",
" 'script_url': 'http://static.criteo.net/js/ld/ld.js'},\n",
" {'json': {'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:CACHE_BUSTING': '\"263385\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:HIT_DATE': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:LAST_HIT_DATE': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:LAST_VISIT': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:PAGE_NUMBER': '\"1\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:PROJECT_ID': '269',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:RECORDING': '\"0\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SCROLL_RATE': '7',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SESSION_NUMBER': '\"1\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:USER_ID': '\"d8151d4d-1a7c-a810-c2dd-5b1cc3a0f286\"',\n",
" 'CSStorageData:stores:412876:eventsStore:queues:events:0': '[0,3485,1366,697]',\n",
" 'CSStorageData:timeStamp': '1513385259071'},\n",
" 'script_url': 'http://static.criteo.net/js/ld/ld.js'},\n",
" {'json': {'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:CACHE_BUSTING': '\"263385\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:HIT_DATE': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:LAST_HIT_DATE': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:LAST_VISIT': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:PAGE_NUMBER': '\"1\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:PROJECT_ID': '269',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:RECORDING': '\"0\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SCROLL_RATE': '7',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SESSION_NUMBER': '\"1\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:USER_ID': '\"d8151d4d-1a7c-a810-c2dd-5b1cc3a0f286\"',\n",
" 'CSStorageData:stores:412876:eventsStore:queues:events:0': '[0,3485,1366,697]',\n",
" 'CSStorageData:timeStamp': '1513385259071'},\n",
" 'script_url': 'http://static.criteo.net/js/ld/ld.js'},\n",
" {'json': {'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:CACHE_BUSTING': '\"263385\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:HIT_DATE': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:LAST_HIT_DATE': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:LAST_VISIT': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:PAGE_NUMBER': '\"1\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:PROJECT_ID': '269',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:RECORDING': '\"0\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SCROLL_RATE': '7',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SESSION_NUMBER': '\"1\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:USER_ID': '\"d8151d4d-1a7c-a810-c2dd-5b1cc3a0f286\"',\n",
" 'CSStorageData:stores:412876:eventsStore:queues:events:0': '[0,3485,1366,697]',\n",
" 'CSStorageData:timeStamp': '1513385259071'},\n",
" 'script_url': 'https://pmcdn.staticpmrk.com/rakuten-static-deliver/app/397.0.1/20170919/static/front/libraries/ral/ral-1.0.20.js'},\n",
" {'json': {'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:CACHE_BUSTING': '\"263385\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:HIT_DATE': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:LAST_HIT_DATE': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:LAST_VISIT': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:PAGE_NUMBER': '\"1\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:PROJECT_ID': '269',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:RECORDING': '\"0\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SCROLL_RATE': '7',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SESSION_NUMBER': '\"1\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:USER_ID': '\"d8151d4d-1a7c-a810-c2dd-5b1cc3a0f286\"',\n",
" 'CSStorageData:stores:412876:eventsStore:queues:events:0': '[0,3485,1366,697]',\n",
" 'CSStorageData:timeStamp': '1513385259071'},\n",
" 'script_url': 'https://pmcdn.staticpmrk.com/rakuten-static-deliver/app/397.0.1/20170919/static/front/libraries/ral/ral-1.0.20.js'},\n",
" {'json': {'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:CACHE_BUSTING': '\"263385\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:HIT_DATE': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:LAST_HIT_DATE': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:LAST_VISIT': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:PAGE_NUMBER': '\"1\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:PROJECT_ID': '269',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:RECORDING': '\"0\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SCROLL_RATE': '7',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SESSION_NUMBER': '\"1\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:USER_ID': '\"d8151d4d-1a7c-a810-c2dd-5b1cc3a0f286\"',\n",
" 'CSStorageData:stores:412876:eventsStore:queues:events:0': '[0,3485,1366,697]',\n",
" 'CSStorageData:timeStamp': '1513385259071'},\n",
" 'script_url': 'https://pmcdn.staticpmrk.com/rakuten-static-deliver/app/397.0.1/20170919/static/front/libraries/ral/ral-1.0.20.js'},\n",
" {'json': {'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:CACHE_BUSTING': '\"263385\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:HIT_DATE': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:LAST_HIT_DATE': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:LAST_VISIT': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:PAGE_NUMBER': '\"1\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:PROJECT_ID': '269',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:RECORDING': '\"0\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SCROLL_RATE': '7',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SESSION_NUMBER': '\"1\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:USER_ID': '\"d8151d4d-1a7c-a810-c2dd-5b1cc3a0f286\"',\n",
" 'CSStorageData:stores:412876:eventsStore:queues:events:0': '[0,3485,1366,697]',\n",
" 'CSStorageData:timeStamp': '1513385259071'},\n",
" 'script_url': 'https://pmcdn.staticpmrk.com/rakuten-static-deliver/app/397.0.1/20170919/static/front/libraries/ral/ral-1.0.20.js'},\n",
" {'json': {'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:CACHE_BUSTING': '\"263385\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:HIT_DATE': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:LAST_HIT_DATE': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:LAST_VISIT': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:PAGE_NUMBER': '\"1\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:PROJECT_ID': '269',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:RECORDING': '\"0\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SCROLL_RATE': '7',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SESSION_NUMBER': '\"1\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:USER_ID': '\"d8151d4d-1a7c-a810-c2dd-5b1cc3a0f286\"',\n",
" 'CSStorageData:stores:412876:eventsStore:queues:events:0': '[0,3485,1366,697]',\n",
" 'CSStorageData:timeStamp': '1513385259071'},\n",
" 'script_url': 'https://pmcdn.staticpmrk.com/rakuten-static-deliver/app/397.0.1/20170919/static/front/libraries/ral/ral-1.0.20.js'},\n",
" {'json': {'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:CACHE_BUSTING': '\"263385\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:HIT_DATE': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:LAST_HIT_DATE': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:LAST_VISIT': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:PAGE_NUMBER': '\"1\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:PROJECT_ID': '269',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:RECORDING': '\"0\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SCROLL_RATE': '7',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SESSION_NUMBER': '\"1\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:USER_ID': '\"d8151d4d-1a7c-a810-c2dd-5b1cc3a0f286\"',\n",
" 'CSStorageData:stores:412876:eventsStore:queues:events:0': '[0,3485,1366,697]',\n",
" 'CSStorageData:timeStamp': '1513385259071',\n",
" 'RALROFL': '{\"ltm\":\"2017-12-16 00:47:43\",\"acc\":10,\"cntln\":\"fr\",\"reqc\":\"success\",\"pgt\":\"nav\",\"pgn\":\"nav\",\"aid\":5,\"cp\":{\"rg\":null,\"buyer\":\"2\",\"rsp\":5,\"ekm\":\"\",\"usergenre\":null,\"usertrackinggroup\":null,\"club_status\":\"\"},\"pgl\":\"PC\",\"icategories\":\"jardin,mobilier-de-jardin\",\"cat1\":\"jardin\",\"cat2\":\"mobilier-de-jardin\",\"etype\":\"pv\",\"url\":\"http://www.priceminister.com/nav/jardin_mobilier-de-jardin/f2/Coffre+de+jardin\",\"tid\":\"9c355f85\",\"tzo\":0,\"res\":\"1366x768\",\"jav\":false,\"bln\":\"en-US\",\"ua\":\"Mozilla/5.0 (X11; Linux x86_64; rv:52.0) Gecko/20100101 Firefox/52.0\",\"online\":true,\"ver\":\"1.0.20d\",\"rqtime\":1041,\"ldtime\":2446,\"astime\":4157,\"navtype\":0,\"ifr\":0,\"pgid\":\"d61245cc156d369b\",\"cks\":\"4fd32bc9-4a6d-4d63-88b6-bdefd08f3c22\"}'},\n",
" 'script_url': 'https://pmcdn.staticpmrk.com/rakuten-static-deliver/app/397.0.1/20170919/static/front/libraries/ral/ral-1.0.20.js'},\n",
" {'json': {'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:CACHE_BUSTING': '\"263385\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:HIT_DATE': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:LAST_HIT_DATE': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:LAST_VISIT': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:PAGE_NUMBER': '\"1\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:PROJECT_ID': '269',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:RECORDING': '\"0\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SCROLL_RATE': '7',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SESSION_NUMBER': '\"1\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:USER_ID': '\"d8151d4d-1a7c-a810-c2dd-5b1cc3a0f286\"',\n",
" 'CSStorageData:stores:412876:eventsStore:queues:events:0': '[0,3485,1366,697]',\n",
" 'CSStorageData:timeStamp': '1513385259071',\n",
" 'RALROFL': '{\"ltm\":\"2017-12-16 00:47:43\",\"acc\":10,\"cntln\":\"fr\",\"reqc\":\"success\",\"pgt\":\"nav\",\"pgn\":\"nav\",\"aid\":5,\"cp\":{\"rg\":null,\"buyer\":\"2\",\"rsp\":5,\"ekm\":\"\",\"usergenre\":null,\"usertrackinggroup\":null,\"club_status\":\"\"},\"pgl\":\"PC\",\"icategories\":\"jardin,mobilier-de-jardin\",\"cat1\":\"jardin\",\"cat2\":\"mobilier-de-jardin\",\"etype\":\"pv\",\"url\":\"http://www.priceminister.com/nav/jardin_mobilier-de-jardin/f2/Coffre+de+jardin\",\"tid\":\"9c355f85\",\"tzo\":0,\"res\":\"1366x768\",\"jav\":false,\"bln\":\"en-US\",\"ua\":\"Mozilla/5.0 (X11; Linux x86_64; rv:52.0) Gecko/20100101 Firefox/52.0\",\"online\":true,\"ver\":\"1.0.20d\",\"rqtime\":1041,\"ldtime\":2446,\"astime\":4157,\"navtype\":0,\"ifr\":0,\"pgid\":\"d61245cc156d369b\",\"cks\":\"4fd32bc9-4a6d-4d63-88b6-bdefd08f3c22\"}'},\n",
" 'script_url': 'https://pmcdn.staticpmrk.com/rakuten-static-deliver/app/397.0.1/20170919/static/front/libraries/ral/ral-1.0.20.js'},\n",
" {'json': {'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:CACHE_BUSTING': '\"263385\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:HIT_DATE': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:LAST_HIT_DATE': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:LAST_VISIT': '\"1513385259\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:PAGE_NUMBER': '\"1\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:PROJECT_ID': '269',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:RECORDING': '\"0\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SCROLL_RATE': '7',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:SESSION_NUMBER': '\"1\"',\n",
" 'CSStorageData:stores:412876:eventsStore:maps:eventsMetaData:USER_ID': '\"d8151d4d-1a7c-a810-c2dd-5b1cc3a0f286\"',\n",
" 'CSStorageData:stores:412876:eventsStore:queues:events:0': '[0,3485,1366,697]',\n",
" 'CSStorageData:timeStamp': '1513385259071',\n",
" 'RALROFL': '{\"ltm\":\"2017-12-16 00:47:43\",\"acc\":10,\"cntln\":\"fr\",\"reqc\":\"success\",\"pgt\":\"nav\",\"pgn\":\"nav\",\"aid\":5,\"cp\":{\"rg\":null,\"buyer\":\"2\",\"rsp\":5,\"ekm\":\"\",\"usergenre\":null,\"usertrackinggroup\":null,\"club_status\":\"\"},\"pgl\":\"PC\",\"icategories\":\"jardin,mobilier-de-jardin\",\"cat1\":\"jardin\",\"cat2\":\"mobilier-de-jardin\",\"etype\":\"pv\",\"url\":\"http://www.priceminister.com/nav/jardin_mobilier-de-jardin/f2/Coffre+de+jardin\",\"tid\":\"9c355f85\",\"tzo\":0,\"res\":\"1366x768\",\"jav\":false,\"bln\":\"en-US\",\"ua\":\"Mozilla/5.0 (X11; Linux x86_64; rv:52.0) Gecko/20100101 Firefox/52.0\",\"online\":true,\"ver\":\"1.0.20d\",\"rqtime\":1041,\"ldtime\":2446,\"astime\":4157,\"navtype\":0,\"ifr\":0,\"pgid\":\"d61245cc156d369b\",\"cks\":\"4fd32bc9-4a6d-4d63-88b6-bdefd08f3c22\"}'},\n",
" 'script_url': 'https://pmcdn.staticpmrk.com/rakuten-static-deliver/app/397.0.1/20170919/static/front/libraries/ral/ral-1.0.20.js'},\n",
" {'json': {'_at.cww': '{\"value\":true,\"expires\":1513386547738}',\n",
" 'at-lojson-cache-xa-4d83f5dd760fecd5': '{\"config\":null,\"perConfig\":{}}',\n",
" 'at-rand': '0.6129927146038678',\n",
" 'google_experiment_mod': '570',\n",
" 'google_pub_config': '{\"sraConfigs\":{\"2\":{\"sraTimeout\":60000},\"4\":{\"sraTimeout\":60000}}}'},\n",
" 'script_url': 'https://s7.addthis.com/js/250/addthis_widget.js#pubid=xa-4d83f5dd760fecd5'}]"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"test = result.loc[(result['symbol'] == \"window.localStorage\")]\n",
"localStorage = []\n",
"for i, row in test.iterrows():\n",
" localStorage.append({'json': json.loads(row['value']), 'script_url':row['script_url']})\n",
"\n",
"localStorage"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.5.2"
}
},
"nbformat": 4,
"nbformat_minor": 2
}

47
utils/simple_cache.py Normal file
Просмотреть файл

@ -0,0 +1,47 @@
import os
import json
"""
A simple cache for storing data in a json file
Current use case: is for reducing the amount of requests to Cookiepedia
"""
class Simple_Json_Cache:
file_name = None
file_data = {}
def __init__(self, cache_name):
""" creates cache storage location and loads data from disk if available """
self.file_name = '.cache/{}.json'.format(cache_name)
if not os.path.exists(os.path.dirname(self.file_name)):
os.makedirs(os.path.dirname(self.file_name))
if os.path.isfile(self.file_name):
self.file_data = self.get_cache_file()
def get_value(self, key):
""" gets value from in memory cache """
return self.file_data.get(key, None)
def set_value(self, key, value):
""" adds value to in memory cache """
self.file_data[key] = value
def dump(self):
""" pushes memory cache into file """
with open(self.file_name, 'w+') as out_file:
json.dump(self.file_data, out_file)
def get_cache_file(self):
""" returns the cache file. Mainly for the class and debugging purpose """
data = {}
with open(self.file_name) as json_data:
data = json.load(json_data)
return data
def get_cache_memory(self):
""" returns the current memory as a dict for looping over """
return self.file_data