BDC Sync
This commit is contained in:
Buck Woody
Родитель
83c72df6f3
Коммит
f53f893048
Двоичный файл не отображается.
|
|
@ -1,5 +1,8 @@
|
|||
{
|
||||
"workbench.colorCustomizations": {
|
||||
"activityBar.background": "#05351D",
|
||||
"titleBar.activeBackground": "#074A29",
|
||||
"titleBar.activeForeground": "#EFFEF6"
|
||||
},
|
||||
"python.linting.pylintEnabled": true
|
||||
}
|
||||
|
|
@ -328,3 +328,4 @@ ASALocalRun/
|
|||
|
||||
# MFractors (Xamarin productivity tool) working folder
|
||||
.mfractor/
|
||||
.azuredatastudio/settings.json
|
||||
|
|
|
|||
|
|
@ -1,10 +0,0 @@
|
|||
{
|
||||
"folders": [
|
||||
{
|
||||
"path": "D:\\OneDrive - Microsoft\\Courses\\sqlworkshopsrepo"
|
||||
}
|
||||
],
|
||||
"settings": {
|
||||
"workbench.colorCustomizations": {}
|
||||
}
|
||||
}
|
||||
Различия файлов скрыты, потому что одна или несколько строк слишком длинны
|
|
@ -33,7 +33,7 @@
|
|||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/html": "Total execution time: 00:00:00.0014441"
|
||||
"text/html": "Total execution time: 00:00:00.0921343"
|
||||
},
|
||||
"metadata": {}
|
||||
},
|
||||
|
|
@ -47,16 +47,16 @@
|
|||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/html": "Total execution time: 00:00:00.8070917"
|
||||
"text/html": "Total execution time: 00:00:00.8982749"
|
||||
},
|
||||
"metadata": {}
|
||||
}
|
||||
],
|
||||
"execution_count": 9
|
||||
"execution_count": 1
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": "/* Create External Data Source to the Storage Pool */\r\nIF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool')\r\nBEGIN\r\n CREATE EXTERNAL DATA SOURCE SqlStoragePool\r\n WITH (LOCATION = 'sqlhdfs://service-master-pool:50070');\r\nEND",
|
||||
"source": "/* Create External Data Source to the Storage Pool */\r\nIF NOT EXISTS(SELECT * FROM sys.external_data_sources WHERE name = 'SqlStoragePool')\r\nBEGIN\r\n CREATE EXTERNAL DATA SOURCE SqlStoragePool\r\n WITH (LOCATION = 'sqlhdfs://nmnode-0-svc:50070');\r\nEND",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
|
|
@ -69,12 +69,12 @@
|
|||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/html": "Total execution time: 00:00:00.6027008"
|
||||
"text/html": "Total execution time: 00:00:00.3378921"
|
||||
},
|
||||
"metadata": {}
|
||||
}
|
||||
],
|
||||
"execution_count": 10
|
||||
"execution_count": 5
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
|
|
@ -91,12 +91,12 @@
|
|||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/html": "Total execution time: 00:00:01.0574098"
|
||||
"text/html": "Total execution time: 00:00:00.3487378"
|
||||
},
|
||||
"metadata": {}
|
||||
}
|
||||
],
|
||||
"execution_count": 11
|
||||
"execution_count": 6
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
|
|
@ -113,14 +113,14 @@
|
|||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/html": "Total execution time: 00:00:07.1517465"
|
||||
"text/html": "Total execution time: 00:00:03.0799860"
|
||||
},
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"output_type": "execute_result",
|
||||
"metadata": {},
|
||||
"execution_count": 12,
|
||||
"execution_count": 1,
|
||||
"data": {
|
||||
"application/vnd.dataresource+json": {
|
||||
"schema": {
|
||||
|
|
@ -193,7 +193,7 @@
|
|||
}
|
||||
}
|
||||
],
|
||||
"execution_count": 12
|
||||
"execution_count": 1
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
|
|
@ -210,7 +210,7 @@
|
|||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/html": "Total execution time: 00:00:00.0020147"
|
||||
"text/html": "Total execution time: 00:00:00.0415968"
|
||||
},
|
||||
"metadata": {}
|
||||
},
|
||||
|
|
@ -224,14 +224,14 @@
|
|||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/html": "Total execution time: 00:00:05.3518348"
|
||||
"text/html": "Total execution time: 00:00:02.0358524"
|
||||
},
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"output_type": "execute_result",
|
||||
"metadata": {},
|
||||
"execution_count": 13,
|
||||
"execution_count": 2,
|
||||
"data": {
|
||||
"application/vnd.dataresource+json": {
|
||||
"schema": {
|
||||
|
|
@ -291,7 +291,7 @@
|
|||
}
|
||||
}
|
||||
],
|
||||
"execution_count": 13
|
||||
"execution_count": 2
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@
|
|||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/html": "Total execution time: 00:00:00.0014288"
|
||||
"text/html": "Total execution time: 00:00:00.0414641"
|
||||
},
|
||||
"metadata": {}
|
||||
},
|
||||
|
|
@ -47,12 +47,12 @@
|
|||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/html": "Total execution time: 00:00:00.6209978"
|
||||
"text/html": "Total execution time: 00:00:00.0663092"
|
||||
},
|
||||
"metadata": {}
|
||||
}
|
||||
],
|
||||
"execution_count": 8
|
||||
"execution_count": 1
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
|
|
@ -69,12 +69,12 @@
|
|||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/html": "Total execution time: 00:00:00.9618554"
|
||||
"text/html": "Total execution time: 00:00:09.9645892"
|
||||
},
|
||||
"metadata": {}
|
||||
}
|
||||
],
|
||||
"execution_count": 9
|
||||
"execution_count": 2
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
|
|
@ -91,12 +91,12 @@
|
|||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/html": "Total execution time: 00:00:00.0011489"
|
||||
"text/html": "Total execution time: 00:00:00.4667564"
|
||||
},
|
||||
"metadata": {}
|
||||
}
|
||||
],
|
||||
"execution_count": 10
|
||||
"execution_count": 3
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
|
|
@ -113,7 +113,7 @@
|
|||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/html": "Total execution time: 00:01:35.0381191"
|
||||
"text/html": "Total execution time: 00:00:52.7335665"
|
||||
},
|
||||
"metadata": {}
|
||||
}
|
||||
|
|
@ -148,7 +148,6 @@
|
|||
},
|
||||
{
|
||||
"output_type": "execute_result",
|
||||
"metadata": {},
|
||||
"execution_count": 13,
|
||||
"data": {
|
||||
"application/vnd.dataresource+json": {
|
||||
|
|
@ -166,11 +165,11 @@
|
|||
]
|
||||
},
|
||||
"text/html": "<table><tr><th>(No column name)</th></tr><tr><td>6770549</td></tr></table>"
|
||||
}
|
||||
},
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"output_type": "execute_result",
|
||||
"metadata": {},
|
||||
"execution_count": 13,
|
||||
"data": {
|
||||
"application/vnd.dataresource+json": {
|
||||
|
|
@ -280,7 +279,8 @@
|
|||
]
|
||||
},
|
||||
"text/html": "<table><tr><th>wcs_click_date_sk</th><th>wcs_click_time_sk</th><th>wcs_sales_sk</th><th>wcs_item_sk</th><th>wcs_web_page_sk</th><th>wcs_user_sk</th></tr><tr><td>37735</td><td>67197</td><td>NULL</td><td>8769</td><td>53</td><td>NULL</td></tr><tr><td>37735</td><td>66368</td><td>NULL</td><td>4710</td><td>53</td><td>NULL</td></tr><tr><td>37735</td><td>29134</td><td>NULL</td><td>2568</td><td>53</td><td>NULL</td></tr><tr><td>37736</td><td>62336</td><td>NULL</td><td>11924</td><td>53</td><td>NULL</td></tr><tr><td>37736</td><td>61470</td><td>NULL</td><td>3914</td><td>53</td><td>NULL</td></tr><tr><td>37736</td><td>6060</td><td>NULL</td><td>16280</td><td>53</td><td>NULL</td></tr><tr><td>37736</td><td>32023</td><td>NULL</td><td>12267</td><td>53</td><td>NULL</td></tr><tr><td>37736</td><td>49579</td><td>NULL</td><td>15327</td><td>53</td><td>NULL</td></tr><tr><td>37736</td><td>56435</td><td>NULL</td><td>15305</td><td>53</td><td>NULL</td></tr><tr><td>37736</td><td>7883</td><td>NULL</td><td>4853</td><td>53</td><td>NULL</td></tr></table>"
|
||||
}
|
||||
},
|
||||
"metadata": {}
|
||||
}
|
||||
],
|
||||
"execution_count": 13
|
||||
|
|
|
|||
|
|
@ -28,22 +28,22 @@
|
|||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "stream",
|
||||
"name": "stdout",
|
||||
"text": "Starting Spark application\n"
|
||||
"text": "Starting Spark application\n",
|
||||
"output_type": "stream"
|
||||
},
|
||||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/plain": "<IPython.core.display.HTML object>",
|
||||
"text/html": "<table>\n<tr><th>ID</th><th>YARN Application ID</th><th>Kind</th><th>State</th><th>Spark UI</th><th>Driver log</th><th>Current session?</th></tr><tr><td>0</td><td>application_1555723257235_0001</td><td>pyspark3</td><td>idle</td><td><a target=\"_blank\" href=\"https://40.118.205.72:30443/gateway/default/yarn/proxy/application_1555723257235_0001/\">Link</a></td><td><a target=\"_blank\" href=\"http://mssql-storage-pool-default-1.service-storage-pool-default.sqlbigdata.svc.cluster.local:8042/node/containerlogs/container_1555723257235_0001_01_000001/root\">Link</a></td><td>✔</td></tr></table>"
|
||||
"text/html": "<table>\n<tr><th>ID</th><th>YARN Application ID</th><th>Kind</th><th>State</th><th>Spark UI</th><th>Driver log</th><th>Current session?</th></tr><tr><td>0</td><td>application_1556393889616_0001</td><td>pyspark3</td><td>idle</td><td><a target=\"_blank\" href=\"https://104.44.132.63:30443/gateway/default/yarn/proxy/application_1556393889616_0001/\">Link</a></td><td><a target=\"_blank\" href=\"http://storage-0-1.storage-0-svc.mssql-cluster.svc.cluster.local:8042/node/containerlogs/container_1556393889616_0001_01_000001/root\">Link</a></td><td>✔</td></tr></table>"
|
||||
},
|
||||
"metadata": {}
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"output_type": "stream",
|
||||
"name": "stdout",
|
||||
"text": "SparkSession available as 'spark'.\n"
|
||||
"text": "SparkSession available as 'spark'.\n",
|
||||
"output_type": "stream"
|
||||
}
|
||||
],
|
||||
"execution_count": 3
|
||||
|
|
@ -52,13 +52,7 @@
|
|||
"cell_type": "code",
|
||||
"source": "# Save results as parquet file and create hive table\r\nresults.write.format(\"parquet\").mode(\"overwrite\").saveAsTable(\"Top_Product_Reviews\")",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"text": "The code failed because of a fatal error:\n\tSession 0 did not start up in 60 seconds..\n\nSome things to try:\na) Make sure Spark has enough available resources for Jupyter to create a Spark context.\nb) Contact your Jupyter administrator to make sure the Spark magics library is configured correctly.\nc) Restart the kernel.\n",
|
||||
"output_type": "stream"
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"execution_count": 4
|
||||
},
|
||||
{
|
||||
|
|
@ -67,12 +61,12 @@
|
|||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "stream",
|
||||
"name": "stdout",
|
||||
"text": "+-------+--------------------+\n|Item_ID| Review|\n+-------+--------------------+\n| 72621|Works fine. Easy ...|\n| 89334|great product to ...|\n| 89335|Next time will go...|\n| 84259|Great Gift Great ...|\n| 84398|After trip to Par...|\n| 66434|Simply the best t...|\n| 66501|This is the exact...|\n| 66587|Not super magnet;...|\n| 66680|Installed as bath...|\n| 66694|Our home was buil...|\n+-------+--------------------+"
|
||||
"text": "+-------+--------------------+\n|Item_ID| Review|\n+-------+--------------------+\n| 72621|Works fine. Easy ...|\n| 89334|great product to ...|\n| 89335|Next time will go...|\n| 84259|Great Gift Great ...|\n| 84398|After trip to Par...|\n| 66434|Simply the best t...|\n| 66501|This is the exact...|\n| 66587|Not super magnet;...|\n| 66680|Installed as bath...|\n| 66694|Our home was buil...|\n+-------+--------------------+",
|
||||
"output_type": "stream"
|
||||
}
|
||||
],
|
||||
"execution_count": 9
|
||||
"execution_count": 5
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
|
|
|
|||
|
|
@ -5,8 +5,13 @@
|
|||
"display_name": "PySpark3"
|
||||
},
|
||||
"language_info": {
|
||||
"name": "python",
|
||||
"version": ""
|
||||
"name": "pyspark3",
|
||||
"mimetype": "text/x-python",
|
||||
"codemirror_mode": {
|
||||
"name": "python",
|
||||
"version": 3
|
||||
},
|
||||
"pygments_lexer": "python3"
|
||||
}
|
||||
},
|
||||
"nbformat_minor": 2,
|
||||
|
|
@ -23,22 +28,22 @@
|
|||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "stream",
|
||||
"name": "stdout",
|
||||
"text": "Starting Spark application\n"
|
||||
"text": "Starting Spark application\n",
|
||||
"output_type": "stream"
|
||||
},
|
||||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/plain": "<IPython.core.display.HTML object>",
|
||||
"text/html": "<table>\n<tr><th>ID</th><th>YARN Application ID</th><th>Kind</th><th>State</th><th>Spark UI</th><th>Driver log</th><th>Current session?</th></tr><tr><td>1</td><td>application_1556023647622_0002</td><td>pyspark3</td><td>idle</td><td><a target=\"_blank\" href=\"https://10.84.56.16:30443/gateway/default/yarn/proxy/application_1556023647622_0002/\">Link</a></td><td><a target=\"_blank\" href=\"http://mssql-storage-pool-default-1.service-storage-pool-default.sqlbigdata.svc.cluster.local:8042/node/containerlogs/container_1556023647622_0002_01_000001/root\">Link</a></td><td>✔</td></tr></table>"
|
||||
"text/html": "<table>\n<tr><th>ID</th><th>YARN Application ID</th><th>Kind</th><th>State</th><th>Spark UI</th><th>Driver log</th><th>Current session?</th></tr><tr><td>1</td><td>application_1556393889616_0002</td><td>pyspark3</td><td>idle</td><td><a target=\"_blank\" href=\"https://104.44.132.63:30443/gateway/default/yarn/proxy/application_1556393889616_0002/\">Link</a></td><td><a target=\"_blank\" href=\"http://storage-0-0.storage-0-svc.mssql-cluster.svc.cluster.local:8042/node/containerlogs/container_1556393889616_0002_01_000001/root\">Link</a></td><td>✔</td></tr></table>"
|
||||
},
|
||||
"metadata": {}
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"output_type": "stream",
|
||||
"name": "stdout",
|
||||
"text": "SparkSession available as 'spark'.\n"
|
||||
"text": "SparkSession available as 'spark'.\n",
|
||||
"output_type": "stream"
|
||||
}
|
||||
],
|
||||
"execution_count": 3
|
||||
|
|
@ -49,9 +54,9 @@
|
|||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "stream",
|
||||
"name": "stdout",
|
||||
"text": "(10000, 74)\n['Survival_In_Days', 'Province', 'Region', 'Trip_Length_Mean', 'Trip_Length_Sigma', 'Trips_Per_Day_Mean', 'Trips_Per_Day_Sigma', 'Battery_Rated_Cycles', 'Manufacture_Month', 'Manufacture_Year', 'Alternator_Efficiency', 'Car_Has_EcoStart', 'Twelve_hourly_temperature_history_for_last_31_days_before_death_last_recording_first', 'Sensor_Reading_1', 'Sensor_Reading_2', 'Sensor_Reading_3', 'Sensor_Reading_4', 'Sensor_Reading_5', 'Sensor_Reading_6', 'Sensor_Reading_7', 'Sensor_Reading_8', 'Sensor_Reading_9', 'Sensor_Reading_10', 'Sensor_Reading_11', 'Sensor_Reading_12', 'Sensor_Reading_13', 'Sensor_Reading_14', 'Sensor_Reading_15', 'Sensor_Reading_16', 'Sensor_Reading_17', 'Sensor_Reading_18', 'Sensor_Reading_19', 'Sensor_Reading_20', 'Sensor_Reading_21', 'Sensor_Reading_22', 'Sensor_Reading_23', 'Sensor_Reading_24', 'Sensor_Reading_25', 'Sensor_Reading_26', 'Sensor_Reading_27', 'Sensor_Reading_28', 'Sensor_Reading_29', 'Sensor_Reading_30', 'Sensor_Reading_31', 'Sensor_Reading_32', 'Sensor_Reading_33', 'Sensor_Reading_34', 'Sensor_Reading_35', 'Sensor_Reading_36', 'Sensor_Reading_37', 'Sensor_Reading_38', 'Sensor_Reading_39', 'Sensor_Reading_40', 'Sensor_Reading_41', 'Sensor_Reading_42', 'Sensor_Reading_43', 'Sensor_Reading_44', 'Sensor_Reading_45', 'Sensor_Reading_46', 'Sensor_Reading_47', 'Sensor_Reading_48', 'Sensor_Reading_49', 'Sensor_Reading_50', 'Sensor_Reading_51', 'Sensor_Reading_52', 'Sensor_Reading_53', 'Sensor_Reading_54', 'Sensor_Reading_55', 'Sensor_Reading_56', 'Sensor_Reading_57', 'Sensor_Reading_58', 'Sensor_Reading_59', 'Sensor_Reading_60', 'Sensor_Reading_61']"
|
||||
"text": "(10000, 74)\n['Survival_In_Days', 'Province', 'Region', 'Trip_Length_Mean', 'Trip_Length_Sigma', 'Trips_Per_Day_Mean', 'Trips_Per_Day_Sigma', 'Battery_Rated_Cycles', 'Manufacture_Month', 'Manufacture_Year', 'Alternator_Efficiency', 'Car_Has_EcoStart', 'Twelve_hourly_temperature_history_for_last_31_days_before_death_last_recording_first', 'Sensor_Reading_1', 'Sensor_Reading_2', 'Sensor_Reading_3', 'Sensor_Reading_4', 'Sensor_Reading_5', 'Sensor_Reading_6', 'Sensor_Reading_7', 'Sensor_Reading_8', 'Sensor_Reading_9', 'Sensor_Reading_10', 'Sensor_Reading_11', 'Sensor_Reading_12', 'Sensor_Reading_13', 'Sensor_Reading_14', 'Sensor_Reading_15', 'Sensor_Reading_16', 'Sensor_Reading_17', 'Sensor_Reading_18', 'Sensor_Reading_19', 'Sensor_Reading_20', 'Sensor_Reading_21', 'Sensor_Reading_22', 'Sensor_Reading_23', 'Sensor_Reading_24', 'Sensor_Reading_25', 'Sensor_Reading_26', 'Sensor_Reading_27', 'Sensor_Reading_28', 'Sensor_Reading_29', 'Sensor_Reading_30', 'Sensor_Reading_31', 'Sensor_Reading_32', 'Sensor_Reading_33', 'Sensor_Reading_34', 'Sensor_Reading_35', 'Sensor_Reading_36', 'Sensor_Reading_37', 'Sensor_Reading_38', 'Sensor_Reading_39', 'Sensor_Reading_40', 'Sensor_Reading_41', 'Sensor_Reading_42', 'Sensor_Reading_43', 'Sensor_Reading_44', 'Sensor_Reading_45', 'Sensor_Reading_46', 'Sensor_Reading_47', 'Sensor_Reading_48', 'Sensor_Reading_49', 'Sensor_Reading_50', 'Sensor_Reading_51', 'Sensor_Reading_52', 'Sensor_Reading_53', 'Sensor_Reading_54', 'Sensor_Reading_55', 'Sensor_Reading_56', 'Sensor_Reading_57', 'Sensor_Reading_58', 'Sensor_Reading_59', 'Sensor_Reading_60', 'Sensor_Reading_61']",
|
||||
"output_type": "stream"
|
||||
}
|
||||
],
|
||||
"execution_count": 4
|
||||
|
|
@ -69,9 +74,9 @@
|
|||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "stream",
|
||||
"name": "stdout",
|
||||
"text": "['Trip_Length_Mean', 'Trip_Length_Sigma', 'Trips_Per_Day_Mean', 'Trips_Per_Day_Sigma', 'Battery_Rated_Cycles', 'Alternator_Efficiency', 'Car_Has_EcoStart', 'Twelve_hourly_temperature_history_for_last_31_days_before_death_last_recording_first', 'Sensor_Reading_1', 'Sensor_Reading_2', 'Sensor_Reading_3', 'Sensor_Reading_4', 'Sensor_Reading_5', 'Sensor_Reading_6', 'Sensor_Reading_7', 'Sensor_Reading_8', 'Sensor_Reading_9', 'Sensor_Reading_10', 'Sensor_Reading_11', 'Sensor_Reading_12', 'Sensor_Reading_13', 'Sensor_Reading_14', 'Sensor_Reading_15', 'Sensor_Reading_16', 'Sensor_Reading_17', 'Sensor_Reading_18', 'Sensor_Reading_19', 'Sensor_Reading_20', 'Sensor_Reading_21', 'Sensor_Reading_22', 'Sensor_Reading_23', 'Sensor_Reading_24', 'Sensor_Reading_25', 'Sensor_Reading_26', 'Sensor_Reading_27', 'Sensor_Reading_28', 'Sensor_Reading_29', 'Sensor_Reading_30', 'Sensor_Reading_31', 'Sensor_Reading_32', 'Sensor_Reading_33', 'Sensor_Reading_34', 'Sensor_Reading_35', 'Sensor_Reading_36', 'Sensor_Reading_37', 'Sensor_Reading_38', 'Sensor_Reading_39', 'Sensor_Reading_40', 'Sensor_Reading_41', 'Sensor_Reading_42', 'Sensor_Reading_43', 'Sensor_Reading_44', 'Sensor_Reading_45', 'Sensor_Reading_46', 'Sensor_Reading_47', 'Sensor_Reading_48', 'Sensor_Reading_49', 'Sensor_Reading_50', 'Sensor_Reading_51', 'Sensor_Reading_52', 'Sensor_Reading_53', 'Sensor_Reading_54', 'Sensor_Reading_55', 'Sensor_Reading_56', 'Sensor_Reading_57', 'Sensor_Reading_58', 'Sensor_Reading_59', 'Sensor_Reading_60', 'Sensor_Reading_61']"
|
||||
"text": "['Trip_Length_Mean', 'Trip_Length_Sigma', 'Trips_Per_Day_Mean', 'Trips_Per_Day_Sigma', 'Battery_Rated_Cycles', 'Alternator_Efficiency', 'Car_Has_EcoStart', 'Twelve_hourly_temperature_history_for_last_31_days_before_death_last_recording_first', 'Sensor_Reading_1', 'Sensor_Reading_2', 'Sensor_Reading_3', 'Sensor_Reading_4', 'Sensor_Reading_5', 'Sensor_Reading_6', 'Sensor_Reading_7', 'Sensor_Reading_8', 'Sensor_Reading_9', 'Sensor_Reading_10', 'Sensor_Reading_11', 'Sensor_Reading_12', 'Sensor_Reading_13', 'Sensor_Reading_14', 'Sensor_Reading_15', 'Sensor_Reading_16', 'Sensor_Reading_17', 'Sensor_Reading_18', 'Sensor_Reading_19', 'Sensor_Reading_20', 'Sensor_Reading_21', 'Sensor_Reading_22', 'Sensor_Reading_23', 'Sensor_Reading_24', 'Sensor_Reading_25', 'Sensor_Reading_26', 'Sensor_Reading_27', 'Sensor_Reading_28', 'Sensor_Reading_29', 'Sensor_Reading_30', 'Sensor_Reading_31', 'Sensor_Reading_32', 'Sensor_Reading_33', 'Sensor_Reading_34', 'Sensor_Reading_35', 'Sensor_Reading_36', 'Sensor_Reading_37', 'Sensor_Reading_38', 'Sensor_Reading_39', 'Sensor_Reading_40', 'Sensor_Reading_41', 'Sensor_Reading_42', 'Sensor_Reading_43', 'Sensor_Reading_44', 'Sensor_Reading_45', 'Sensor_Reading_46', 'Sensor_Reading_47', 'Sensor_Reading_48', 'Sensor_Reading_49', 'Sensor_Reading_50', 'Sensor_Reading_51', 'Sensor_Reading_52', 'Sensor_Reading_53', 'Sensor_Reading_54', 'Sensor_Reading_55', 'Sensor_Reading_56', 'Sensor_Reading_57', 'Sensor_Reading_58', 'Sensor_Reading_59', 'Sensor_Reading_60', 'Sensor_Reading_61']",
|
||||
"output_type": "stream"
|
||||
}
|
||||
],
|
||||
"execution_count": 6
|
||||
|
|
@ -82,9 +87,9 @@
|
|||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "stream",
|
||||
"name": "stdout",
|
||||
"text": "GradientBoostingRegressor(alpha=0.9, criterion='friedman_mse', init=None,\n learning_rate=0.1, loss='ls', max_depth=3, max_features=None,\n max_leaf_nodes=None, min_impurity_decrease=0.0,\n min_impurity_split=None, min_samples_leaf=1,\n min_samples_split=2, min_weight_fraction_leaf=0.0,\n n_estimators=100, n_iter_no_change=None, presort='auto',\n random_state=None, subsample=1.0, tol=0.0001,\n validation_fraction=0.1, verbose=0, warm_start=False)"
|
||||
"text": "GradientBoostingRegressor(alpha=0.9, criterion='friedman_mse', init=None,\n learning_rate=0.1, loss='ls', max_depth=3, max_features=None,\n max_leaf_nodes=None, min_impurity_decrease=0.0,\n min_impurity_split=None, min_samples_leaf=1,\n min_samples_split=2, min_weight_fraction_leaf=0.0,\n n_estimators=100, n_iter_no_change=None, presort='auto',\n random_state=None, subsample=1.0, tol=0.0001,\n validation_fraction=0.1, verbose=0, warm_start=False)",
|
||||
"output_type": "stream"
|
||||
}
|
||||
],
|
||||
"execution_count": 7
|
||||
|
|
@ -95,9 +100,9 @@
|
|||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "stream",
|
||||
"name": "stdout",
|
||||
"text": "array([1323.39791998])"
|
||||
"text": "array([1323.39791998])",
|
||||
"output_type": "stream"
|
||||
}
|
||||
],
|
||||
"execution_count": 8
|
||||
|
|
@ -108,9 +113,9 @@
|
|||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "stream",
|
||||
"name": "stdout",
|
||||
"text": "array([1472.91111228, 1340.08897725, 1421.38601032, 1473.79033215,\n 1651.66584142, 1412.85617044, 1842.81351408, 1264.22762055,\n 1930.45602533, 1681.86345995])"
|
||||
"text": "array([1472.91111228, 1340.08897725, 1421.38601032, 1473.79033215,\n 1651.66584142, 1412.85617044, 1842.81351408, 1264.22762055,\n 1930.45602533, 1681.86345995])",
|
||||
"output_type": "stream"
|
||||
}
|
||||
],
|
||||
"execution_count": 9
|
||||
|
|
@ -120,7 +125,7 @@
|
|||
"source": "# prepare one data frame that includes predictions for each vehicle \r\nscored_data = test_data \r\nscored_data[\"Estimated_Battery_Life\"] = battery_life_predictions \r\ndf_scored = spark.createDataFrame(scored_data) \r\n# Write out the scored data: \r\ndf_scored.coalesce(1).write.option(\"header\", \"true\").csv(\"/pdm\") ",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"execution_count": 13
|
||||
"execution_count": 10
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
|
|
@ -128,12 +133,12 @@
|
|||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "stream",
|
||||
"name": "stdout",
|
||||
"text": "/tmp/nm-local-dir/usercache/root/appcache/application_1555723257235_0003/container_1555723257235_0003_01_000001\n['test.pkl', 'pdm.pkl', 'Jetty_0_0_0_0_8042_node____19tj0x', 'nm-local-dir', 'Jetty_localhost_44079_datanode____j8a1yx', 'test.txt', 'tmppboajmm2', 'hsperfdata_root', 'install.sh']"
|
||||
"text": "/tmp/nm-local-dir/usercache/root/appcache/application_1556393889616_0002/container_1556393889616_0002_01_000001\n['hsperfdata_root', 'Jetty_localhost_45131_datanode____.y2vugb', 'nm-local-dir', 'Jetty_0_0_0_0_8042_node____19tj0x', 'tmp0c7_cdnk', 'pdm.pkl']",
|
||||
"output_type": "stream"
|
||||
}
|
||||
],
|
||||
"execution_count": 14
|
||||
"execution_count": 11
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
|
|
|
|||
|
|
@ -3,49 +3,49 @@
|
|||
<Items>
|
||||
<LogicalFolder Name="Connections" Type="2" Sorted="true">
|
||||
<Items>
|
||||
<ConnectionNode Name="40.112.178.197,31433:sa">
|
||||
<Created>2019-04-25T08:01:42.4813296-04:00</Created>
|
||||
<ConnectionNode Name="104.44.142.150,31433:sa">
|
||||
<Created>2019-04-29T13:21:46.7659137-04:00</Created>
|
||||
<Type>SQL</Type>
|
||||
<Server>40.112.178.197,31433</Server>
|
||||
<Server>104.44.142.150,31433</Server>
|
||||
<UserName>sa</UserName>
|
||||
<Authentication>SQL</Authentication>
|
||||
<InitialDB />
|
||||
<LoginTimeout>30</LoginTimeout>
|
||||
<ExecutionTimeout>0</ExecutionTimeout>
|
||||
<ConnectionProtocol>NotSpecified</ConnectionProtocol>
|
||||
<ApplicationName>Microsoft SQL Server Management Studio - Query</ApplicationName>
|
||||
<ApplicationName />
|
||||
</ConnectionNode>
|
||||
</Items>
|
||||
</LogicalFolder>
|
||||
<LogicalFolder Name="Queries" Type="0" Sorted="true">
|
||||
<Items>
|
||||
<FileNode Name="01 - Show Configuration.sql">
|
||||
<AssociatedConnectionMoniker>8c91a03d-f9b4-46c0-a305-b5dcc79ff907:40.112.178.197,31433:False:sa</AssociatedConnectionMoniker>
|
||||
<AssociatedConnSrvName>40.112.178.197,31433</AssociatedConnSrvName>
|
||||
<AssociatedConnectionMoniker>8c91a03d-f9b4-46c0-a305-b5dcc79ff907:104.44.142.150,31433:False:sa</AssociatedConnectionMoniker>
|
||||
<AssociatedConnSrvName>104.44.142.150,31433</AssociatedConnSrvName>
|
||||
<AssociatedConnUserName>sa</AssociatedConnUserName>
|
||||
<FullPath>01 - Show Configuration.sql</FullPath>
|
||||
</FileNode>
|
||||
<FileNode Name="02 - Population Information from WWI.sql">
|
||||
<AssociatedConnectionMoniker>8c91a03d-f9b4-46c0-a305-b5dcc79ff907:40.112.178.197,31433:False:sa</AssociatedConnectionMoniker>
|
||||
<AssociatedConnSrvName>40.112.178.197,31433</AssociatedConnSrvName>
|
||||
<AssociatedConnectionMoniker>8c91a03d-f9b4-46c0-a305-b5dcc79ff907:104.44.142.150,31433:False:sa</AssociatedConnectionMoniker>
|
||||
<AssociatedConnSrvName>104.44.142.150,31433</AssociatedConnSrvName>
|
||||
<AssociatedConnUserName>sa</AssociatedConnUserName>
|
||||
<FullPath>02 - Population Information from WWI.sql</FullPath>
|
||||
</FileNode>
|
||||
<FileNode Name="03 - Sales in WWI.sql">
|
||||
<AssociatedConnectionMoniker>8c91a03d-f9b4-46c0-a305-b5dcc79ff907:40.112.178.197,31433:False:sa</AssociatedConnectionMoniker>
|
||||
<AssociatedConnSrvName>40.112.178.197,31433</AssociatedConnSrvName>
|
||||
<AssociatedConnectionMoniker>8c91a03d-f9b4-46c0-a305-b5dcc79ff907:104.44.142.150,31433:False:sa</AssociatedConnectionMoniker>
|
||||
<AssociatedConnSrvName>104.44.142.150,31433</AssociatedConnSrvName>
|
||||
<AssociatedConnUserName>sa</AssociatedConnUserName>
|
||||
<FullPath>03 - Sales in WWI.sql</FullPath>
|
||||
</FileNode>
|
||||
<FileNode Name="04 - Join to HDFS.sql">
|
||||
<AssociatedConnectionMoniker>8c91a03d-f9b4-46c0-a305-b5dcc79ff907:40.112.178.197,31433:False:sa</AssociatedConnectionMoniker>
|
||||
<AssociatedConnSrvName>40.112.178.197,31433</AssociatedConnSrvName>
|
||||
<AssociatedConnectionMoniker>8c91a03d-f9b4-46c0-a305-b5dcc79ff907:104.44.142.150,31433:False:sa</AssociatedConnectionMoniker>
|
||||
<AssociatedConnSrvName>104.44.142.150,31433</AssociatedConnSrvName>
|
||||
<AssociatedConnUserName>sa</AssociatedConnUserName>
|
||||
<FullPath>04 - Join to HDFS.sql</FullPath>
|
||||
</FileNode>
|
||||
<FileNode Name="05 - Query from Data Pool.sql">
|
||||
<AssociatedConnectionMoniker>8c91a03d-f9b4-46c0-a305-b5dcc79ff907:40.112.178.197,31433:False:sa</AssociatedConnectionMoniker>
|
||||
<AssociatedConnSrvName>40.112.178.197,31433</AssociatedConnSrvName>
|
||||
<AssociatedConnectionMoniker>8c91a03d-f9b4-46c0-a305-b5dcc79ff907:104.44.142.150,31433:False:sa</AssociatedConnectionMoniker>
|
||||
<AssociatedConnSrvName>104.44.142.150,31433</AssociatedConnSrvName>
|
||||
<AssociatedConnUserName>sa</AssociatedConnUserName>
|
||||
<FullPath>05 - Query from Data Pool.sql</FullPath>
|
||||
</FileNode>
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче