dropping the demo
|
@ -0,0 +1,345 @@
|
|||
## Ignore Visual Studio temporary files, build results, and
|
||||
## files generated by popular Visual Studio add-ons.
|
||||
##
|
||||
## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore
|
||||
|
||||
# User-specific files
|
||||
*.suo
|
||||
*.user
|
||||
*.userosscache
|
||||
*.sln.docstates
|
||||
|
||||
.idea/
|
||||
target/
|
||||
.image/
|
||||
build/
|
||||
|
||||
# User-specific files (MonoDevelop/Xamarin Studio)
|
||||
*.userprefs
|
||||
|
||||
# Build results
|
||||
[Dd]ebug/
|
||||
[Dd]ebugPublic/
|
||||
[Rr]elease/
|
||||
[Rr]eleases/
|
||||
x64/
|
||||
x86/
|
||||
bld/
|
||||
[Bb]in/
|
||||
[Oo]bj/
|
||||
[Ll]og/
|
||||
|
||||
# Visual Studio 2015/2017 cache/options directory
|
||||
.vs/
|
||||
.vscode/
|
||||
# Uncomment if you have tasks that create the project's static files in wwwroot
|
||||
#wwwroot/
|
||||
|
||||
# Visual Studio 2017 auto generated files
|
||||
Generated\ Files/
|
||||
|
||||
# MSTest test Results
|
||||
[Tt]est[Rr]esult*/
|
||||
[Bb]uild[Ll]og.*
|
||||
|
||||
# NUNIT
|
||||
*.VisualState.xml
|
||||
TestResult.xml
|
||||
|
||||
# Build Results of an ATL Project
|
||||
[Dd]ebugPS/
|
||||
[Rr]eleasePS/
|
||||
dlldata.c
|
||||
|
||||
# Benchmark Results
|
||||
BenchmarkDotNet.Artifacts/
|
||||
|
||||
# .NET Core
|
||||
project.lock.json
|
||||
project.fragment.lock.json
|
||||
artifacts/
|
||||
**/Properties/launchSettings.json
|
||||
|
||||
# StyleCop
|
||||
StyleCopReport.xml
|
||||
|
||||
# Files built by Visual Studio
|
||||
*_i.c
|
||||
*_p.c
|
||||
*_i.h
|
||||
*.ilk
|
||||
*.meta
|
||||
*.obj
|
||||
*.iobj
|
||||
*.pch
|
||||
*.pdb
|
||||
*.ipdb
|
||||
*.pgc
|
||||
*.pgd
|
||||
*.rsp
|
||||
*.sbr
|
||||
*.tlb
|
||||
*.tli
|
||||
*.tlh
|
||||
*.tmp
|
||||
*.tmp_proj
|
||||
*.log
|
||||
*.vspscc
|
||||
*.vssscc
|
||||
.builds
|
||||
*.pidb
|
||||
*.svclog
|
||||
*.scc
|
||||
|
||||
# Chutzpah Test files
|
||||
_Chutzpah*
|
||||
|
||||
# Visual C++ cache files
|
||||
ipch/
|
||||
*.aps
|
||||
*.ncb
|
||||
*.opendb
|
||||
*.opensdf
|
||||
*.sdf
|
||||
*.cachefile
|
||||
*.VC.db
|
||||
*.VC.VC.opendb
|
||||
out/
|
||||
|
||||
# Visual Studio profiler
|
||||
*.psess
|
||||
*.vsp
|
||||
*.vspx
|
||||
*.sap
|
||||
|
||||
# Visual Studio Trace Files
|
||||
*.e2e
|
||||
|
||||
# TFS 2012 Local Workspace
|
||||
$tf/
|
||||
|
||||
# Guidance Automation Toolkit
|
||||
*.gpState
|
||||
|
||||
# ReSharper is a .NET coding add-in
|
||||
_ReSharper*/
|
||||
*.[Rr]e[Ss]harper
|
||||
*.DotSettings.user
|
||||
|
||||
# JustCode is a .NET coding add-in
|
||||
.JustCode
|
||||
|
||||
# TeamCity is a build add-in
|
||||
_TeamCity*
|
||||
|
||||
# DotCover is a Code Coverage Tool
|
||||
*.dotCover
|
||||
|
||||
# AxoCover is a Code Coverage Tool
|
||||
.axoCover/*
|
||||
!.axoCover/settings.json
|
||||
|
||||
# Visual Studio code coverage results
|
||||
*.coverage
|
||||
*.coveragexml
|
||||
|
||||
# NCrunch
|
||||
_NCrunch_*
|
||||
.*crunch*.local.xml
|
||||
nCrunchTemp_*
|
||||
|
||||
# MightyMoose
|
||||
*.mm.*
|
||||
AutoTest.Net/
|
||||
|
||||
# Web workbench (sass)
|
||||
.sass-cache/
|
||||
|
||||
# Installshield output folder
|
||||
[Ee]xpress/
|
||||
|
||||
# DocProject is a documentation generator add-in
|
||||
DocProject/buildhelp/
|
||||
DocProject/Help/*.HxT
|
||||
DocProject/Help/*.HxC
|
||||
DocProject/Help/*.hhc
|
||||
DocProject/Help/*.hhk
|
||||
DocProject/Help/*.hhp
|
||||
DocProject/Help/Html2
|
||||
DocProject/Help/html
|
||||
|
||||
# Click-Once directory
|
||||
publish/
|
||||
|
||||
# Publish Web Output
|
||||
*.[Pp]ublish.xml
|
||||
*.azurePubxml
|
||||
# Note: Comment the next line if you want to checkin your web deploy settings,
|
||||
# but database connection strings (with potential passwords) will be unencrypted
|
||||
*.pubxml
|
||||
*.publishproj
|
||||
|
||||
# Microsoft Azure Web App publish settings. Comment the next line if you want to
|
||||
# checkin your Azure Web App publish settings, but sensitive information contained
|
||||
# in these scripts will be unencrypted
|
||||
PublishScripts/
|
||||
|
||||
# NuGet Packages
|
||||
*.nupkg
|
||||
# The packages folder can be ignored because of Package Restore
|
||||
**/[Pp]ackages/*
|
||||
# except build/, which is used as an MSBuild target.
|
||||
!**/[Pp]ackages/build/
|
||||
# Uncomment if necessary however generally it will be regenerated when needed
|
||||
#!**/[Pp]ackages/repositories.config
|
||||
# NuGet v3's project.json files produces more ignorable files
|
||||
*.nuget.props
|
||||
*.nuget.targets
|
||||
|
||||
# Microsoft Azure Build Output
|
||||
csx/
|
||||
*.build.csdef
|
||||
|
||||
# Microsoft Azure Emulator
|
||||
ecf/
|
||||
rcf/
|
||||
|
||||
# Windows Store app package directories and files
|
||||
AppPackages/
|
||||
BundleArtifacts/
|
||||
Package.StoreAssociation.xml
|
||||
_pkginfo.txt
|
||||
*.appx
|
||||
|
||||
# Visual Studio cache files
|
||||
# files ending in .cache can be ignored
|
||||
*.[Cc]ache
|
||||
# but keep track of directories ending in .cache
|
||||
!*.[Cc]ache/
|
||||
|
||||
# Others
|
||||
ClientBin/
|
||||
~$*
|
||||
*~
|
||||
*.dbmdl
|
||||
*.dbproj.schemaview
|
||||
*.jfm
|
||||
*.pfx
|
||||
*.publishsettings
|
||||
orleans.codegen.cs
|
||||
|
||||
# Including strong name files can present a security risk
|
||||
# (https://github.com/github/gitignore/pull/2483#issue-259490424)
|
||||
#*.snk
|
||||
|
||||
# Since there are multiple workflows, uncomment next line to ignore bower_components
|
||||
# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
|
||||
#bower_components/
|
||||
|
||||
# RIA/Silverlight projects
|
||||
Generated_Code/
|
||||
|
||||
# Backup & report files from converting an old project file
|
||||
# to a newer Visual Studio version. Backup files are not needed,
|
||||
# because we have git ;-)
|
||||
_UpgradeReport_Files/
|
||||
Backup*/
|
||||
UpgradeLog*.XML
|
||||
UpgradeLog*.htm
|
||||
ServiceFabricBackup/
|
||||
*.rptproj.bak
|
||||
|
||||
# SQL Server files
|
||||
*.mdf
|
||||
*.ldf
|
||||
*.ndf
|
||||
|
||||
# Business Intelligence projects
|
||||
*.rdl.data
|
||||
*.bim.layout
|
||||
*.bim_*.settings
|
||||
*.rptproj.rsuser
|
||||
|
||||
# Microsoft Fakes
|
||||
FakesAssemblies/
|
||||
|
||||
# GhostDoc plugin setting file
|
||||
*.GhostDoc.xml
|
||||
|
||||
# Node.js Tools for Visual Studio
|
||||
.ntvs_analysis.dat
|
||||
node_modules/
|
||||
|
||||
# Visual Studio 6 build log
|
||||
*.plg
|
||||
|
||||
# Visual Studio 6 workspace options file
|
||||
*.opt
|
||||
|
||||
# Visual Studio 6 auto-generated workspace file (contains which files were open etc.)
|
||||
*.vbw
|
||||
|
||||
# Visual Studio LightSwitch build output
|
||||
**/*.HTMLClient/GeneratedArtifacts
|
||||
**/*.DesktopClient/GeneratedArtifacts
|
||||
**/*.DesktopClient/ModelManifest.xml
|
||||
**/*.Server/GeneratedArtifacts
|
||||
**/*.Server/ModelManifest.xml
|
||||
_Pvt_Extensions
|
||||
|
||||
# Paket dependency manager
|
||||
.paket/paket.exe
|
||||
paket-files/
|
||||
|
||||
# FAKE - F# Make
|
||||
.fake/
|
||||
|
||||
# JetBrains Rider
|
||||
.idea/
|
||||
*.sln.iml
|
||||
|
||||
# CodeRush
|
||||
.cr/
|
||||
|
||||
# Python Tools for Visual Studio (PTVS)
|
||||
__pycache__/
|
||||
*.pyc
|
||||
.ipynb_checkpoints/
|
||||
|
||||
# Cake - Uncomment if you are using it
|
||||
# tools/**
|
||||
# !tools/packages.config
|
||||
|
||||
# Tabs Studio
|
||||
*.tss
|
||||
|
||||
# Telerik's JustMock configuration file
|
||||
*.jmconfig
|
||||
|
||||
# BizTalk build output
|
||||
*.btp.cs
|
||||
*.btm.cs
|
||||
*.odx.cs
|
||||
*.xsd.cs
|
||||
|
||||
# OpenCover UI analysis results
|
||||
OpenCover/
|
||||
|
||||
# Azure Stream Analytics local run output
|
||||
ASALocalRun/
|
||||
|
||||
# MSBuild Binary and Structured Log
|
||||
*.binlog
|
||||
|
||||
# NVidia Nsight GPU debugger configuration file
|
||||
*.nvuser
|
||||
|
||||
# MFractors (Xamarin productivity tool) working folder
|
||||
.mfractor/
|
||||
|
||||
#Spark
|
||||
metastore_db/
|
||||
.metals/
|
||||
.scalafmt.conf
|
||||
|
||||
*.zip
|
|
@ -0,0 +1,316 @@
|
|||
import org.apache.spark.sql.Row
|
||||
import org.apache.spark.sql.SparkSession
|
||||
import org.apache.spark.sql.types._
|
||||
|
||||
import java.sql.Timestamp
|
||||
|
||||
// Data from https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page
|
||||
|
||||
/*
|
||||
Yellow Taxi Schema
|
||||
==================
|
||||
Header
|
||||
VendorID,tpep_pickup_datetime,tpep_dropoff_datetime,passenger_count,trip_distance,RatecodeID,store_and_fwd_flag,PULocationID,DOLocationID,payment_type,fare_amount,extra,mta_tax,tip_amount,tolls_amount,improvement_surcharge,total_amount,congestion_surcharge
|
||||
|
||||
VendorID
|
||||
A code indicating the TPEP provider that provided the record.
|
||||
1= Creative Mobile Technologies, LLC; 2= VeriFone Inc.
|
||||
|
||||
tpep_pickup_datetime
|
||||
The date and time when the meter was engaged.
|
||||
|
||||
tpep_dropoff_datetime
|
||||
The date and time when the meter was disengaged.
|
||||
|
||||
Passenger_count
|
||||
The number of passengers in the vehicle.
|
||||
This is a driver-entered value.
|
||||
|
||||
Trip_distance
|
||||
The elapsed trip distance in miles reported by the taximeter.
|
||||
|
||||
PULocationID
|
||||
TLC Taxi Zone in which the taximeter was engaged
|
||||
|
||||
DOLocationID
|
||||
TLC Taxi Zone in which the taximeter was disengaged
|
||||
|
||||
RateCodeID
|
||||
The final rate code in effect at the end of the trip.
|
||||
1= Standard rate 2=JFK 3=Newark 4=Nassau or Westchester 5=Negotiated fare 6=Group ride
|
||||
|
||||
Store_and_fwd_flag
|
||||
This flag indicates whether the trip record was held in vehicle memory before sending to the vendor, aka “store and forward,” because the vehicle did not have a connection to the server.
|
||||
Y= store and forward trip N= not a store and forward trip
|
||||
|
||||
Payment_type
|
||||
A numeric code signifying how the passenger paid for the trip. 1= Credit card 2= Cash 3= No charge 4= Dispute 5= Unknown 6= Voided trip
|
||||
|
||||
Fare_amount
|
||||
The time-and-distance fare calculated by the meter.
|
||||
|
||||
Extra
|
||||
Miscellaneous extras and surcharges. Currently, this only includes the $0.50 and $1 rush hour and overnight charges.
|
||||
|
||||
MTA_tax
|
||||
$0.50 MTA tax that is automatically triggered based on the metered rate in use.
|
||||
|
||||
Improvement_surcharge
|
||||
$0.30 improvement surcharge assessed trips at the flag drop. The improvement surcharge began being levied in 2015.
|
||||
|
||||
Tip_amount
|
||||
Tip amount – This field is automatically populated for credit card tips. Cash tips are not included.
|
||||
|
||||
Tolls_amount
|
||||
Total amount of all tolls paid in trip. Total_amount The total amount charged to passengers. Does not include cash tips.
|
||||
|
||||
*/
|
||||
|
||||
/*
|
||||
Green Taxi Data Schema
|
||||
======================
|
||||
Header
|
||||
VendorID,lpep_pickup_datetime,Lpep_dropoff_datetime,Store_and_fwd_flag,RateCodeID,Pickup_longitude,Pickup_latitude,Dropoff_longitude,Dropoff_latitude,Passenger_count,Trip_distance,Fare_amount,Extra,MTA_tax,Tip_amount,Tolls_amount,Ehail_fee,Total_amount,Payment_type,Trip_type
|
||||
|
||||
VendorID
|
||||
A code indicating the LPEP provider that provided the record.
|
||||
1= Creative Mobile Technologies, LLC; 2= VeriFone Inc. lpep_pickup_datetime The date and time when the meter was engaged. lpep_dropoff_datetime The date and time when the meter was disengaged.
|
||||
|
||||
Passenger_count
|
||||
The number of passengers in the vehicle.
|
||||
This is a driver-entered value. Trip_distance The elapsed trip distance in miles reported by the taximeter. PULocationID TLC Taxi Zone in which the taximeter was engaged DOLocationID TLC Taxi Zone in which the taximeter was disengaged
|
||||
|
||||
RateCodeID
|
||||
The final rate code in effect at the end of the trip.
|
||||
1= Standard rate 2=JFK 3=Newark 4=Nassau or Westchester 5=Negotiated fare 6=Group ride
|
||||
|
||||
Store_and_fwd_flag
|
||||
This flag indicates whether the trip record was held in vehicle memory before sending to the vendor, aka “store and forward,” because the vehicle did not have a connection to the server.
|
||||
Y= store and forward trip N= not a store and forward trip
|
||||
|
||||
Payment_type
|
||||
A numeric code signifying how the passenger paid for the trip. 1= Credit card 2= Cash 3= No charge 4= Dispute 5= Unknown 6= Voided trip
|
||||
|
||||
Fare_amount
|
||||
The time-and-distance fare calculated by the meter.
|
||||
|
||||
Extra
|
||||
Miscellaneous extras and surcharges. Currently, this only includes the $0.50 and $1 rush hour and overnight charges.
|
||||
|
||||
MTA_tax
|
||||
$0.50 MTA tax that is automatically triggered based on the metered rate in use.
|
||||
|
||||
Improvement_surcharge
|
||||
$0.30 improvement surcharge assessed on hailed trips at the flag drop. The improvement surcharge began being levied in 2015.
|
||||
|
||||
Tip_amount
|
||||
Trip amount – This field is automatically populated for credit card tips. Cash tips are not included.
|
||||
|
||||
Tolls_amount
|
||||
Total amount of all tolls paid in trip.
|
||||
|
||||
Total_amount
|
||||
The total amount charged to passengers. Does not include cash tips.
|
||||
|
||||
Trip_type
|
||||
A code indicating whether the trip was a street-hail or a dispatch that is automatically assigned based on the metered rate in use but can be altered by the driver.
|
||||
1= Street-hail 2= Dispatch
|
||||
|
||||
*/
|
||||
|
||||
/*
|
||||
Example Queries
|
||||
|
||||
Query 1
|
||||
SELECT cab_type, count(*) FROM trips_csv GROUP BY cab_type;
|
||||
|
||||
Query 2
|
||||
SELECT passenger_count, avg(total_amount) FROM trips_orc GROUP BY passenger_count
|
||||
|
||||
Query 3
|
||||
SELECT passenger_count, year(pickup_datetime), count(*) FROM trips_orc GROUP BY passenger_count, year(pickup_datetime)
|
||||
|
||||
Query 4
|
||||
SELECT passenger_count, year(pickup_datetime) trip_year, round(trip_distance), count(*) trips FROM trips_orc GROUP BY passenger_count, year(pickup_datetime), round(trip_distance) ORDER BY trip_year, trips desc;
|
||||
|
||||
*/
|
||||
|
||||
/*
|
||||
Schema for Green Taxi
|
||||
|
||||
case class GreenTaxiRide(VendorID: Byte,
|
||||
lpep_pickup_datetime: TimestampType,
|
||||
lpep_dropoff_datetime: TimestampType,
|
||||
Store_and_fwd_flag: Boolean,
|
||||
RateCodeID: Byte,
|
||||
Pickup_longitude: Float,
|
||||
Pickup_latitude: Float,
|
||||
Dropoff_longitude: Float,
|
||||
Dropoff_latitude: Float,
|
||||
Passenger_count: Int,
|
||||
Trip_distance: Float,
|
||||
Fare_amount: Float,
|
||||
Extra: Float,
|
||||
MTA_tax: Float,
|
||||
Tip_amount: FLoat,
|
||||
Tolls_amount: Float,
|
||||
Ehail_fee: Float,
|
||||
Total_amount: Float,
|
||||
Payment_type: Byte,
|
||||
Trip_type: Byte)
|
||||
|
||||
case class YellowTaxiRide (
|
||||
VendorID: Byte,
|
||||
lpep_pickup_datetime: TimestampType,
|
||||
lpep_dropoff_datetime: TimestampType,
|
||||
passenger_count: Int,
|
||||
trip_distance: Float,
|
||||
RateCodeID: Byte,
|
||||
store_and_fwd_flag: Boolean,
|
||||
PULocationID: Int,
|
||||
DOLocationID: Int,
|
||||
payment_type: Byte,
|
||||
fare_amount: Float,
|
||||
extra: Float,
|
||||
mta_tax: Float,
|
||||
tip_amount: FLoat,
|
||||
tolls_amount: Float,
|
||||
improvement_surcharge:Float,
|
||||
total_amount:Float
|
||||
congestion_surcharge: Float
|
||||
|
||||
*/
|
||||
|
||||
object NyTaxiBenchmarks {
|
||||
|
||||
// Converged schema with column projections
|
||||
case class YellowTaxiRide (
|
||||
//VendorID: Byte,
|
||||
pickup_datetime: TimestampType,
|
||||
//dropoff_datetime: TimestampType,
|
||||
passenger_count: Int,
|
||||
trip_distance: Float,
|
||||
//RateCodeID: Byte,
|
||||
//store_and_fwd_flag: Boolean,
|
||||
//PULocationID: Int,
|
||||
//DOLocationID: Int,
|
||||
//payment_type: Byte,
|
||||
//fare_amount: Float,
|
||||
//extra: Float,
|
||||
//mta_tax: Float,
|
||||
//tip_amount: FLoat,
|
||||
//tolls_amount: Float,
|
||||
//improvement_surcharge:Float,
|
||||
total_amount:Float
|
||||
//congestion_surcharge: Float
|
||||
)
|
||||
|
||||
def main(args: Array[String]) {
|
||||
|
||||
val spark = SparkSession.builder().appName("Spark SQL basic example").config("spark.some.config.option", "some-value").getOrCreate()
|
||||
|
||||
// For implicit conversions like converting RDDs to DataFrames
|
||||
import spark.implicits._
|
||||
|
||||
// Print working direcotory
|
||||
System.getProperty("user.dir")
|
||||
|
||||
// Define Programmatic Schema
|
||||
val schema = StructType(
|
||||
List(
|
||||
StructField("pickup_datetime", TimestampType, nullable=true),
|
||||
StructField("passenger_count", IntegerType, nullable=true),
|
||||
StructField("trip_distance", FloatType, nullable=true),
|
||||
StructField("total_amount", FloatType, nullable=true)
|
||||
)
|
||||
)
|
||||
println(schema.toDDL)
|
||||
|
||||
//runBasicCountQuery(spark)
|
||||
|
||||
// Load yellow taxidata
|
||||
val nyYellowTaxiDF = loadYellowTaxiData(spark, schema)
|
||||
|
||||
// Query 1
|
||||
val totRides = spark.sql("select count(*) from nyyellowtaxi")
|
||||
totRides.show()
|
||||
|
||||
// Query 2
|
||||
val avgRides = spark.sql("SELECT passenger_count, avg(total_amount) FROM nyyellowtaxi GROUP BY passenger_count")
|
||||
avgRides.show()
|
||||
|
||||
// Query 3
|
||||
val psgCount = spark.sql("SELECT passenger_count, year(pickup_datetime), count(*) FROM nyyellowtaxi GROUP BY passenger_count, year(pickup_datetime)")
|
||||
psgCount.show()
|
||||
|
||||
// Query 4
|
||||
val psgYearCount = spark.sql("SELECT passenger_count, year(pickup_datetime) trip_year, round(trip_distance), count(*) trips FROM nyyellowtaxi GROUP BY passenger_count, year(pickup_datetime), round(trip_distance) ORDER BY trip_year, trips desc")
|
||||
psgYearCount.show()
|
||||
|
||||
spark.stop()
|
||||
|
||||
}
|
||||
|
||||
def loadYellowTaxiData(spark: SparkSession, schema: StructType): org.apache.spark.sql.DataFrame = {
|
||||
|
||||
// NyTaxi RDD
|
||||
val nyYellowTaxiRDD = spark.sparkContext.textFile("yellow_tripdata_2019-12.csv")
|
||||
nyYellowTaxiRDD.take(5).foreach(println)
|
||||
|
||||
// Convert records of ride to Rows
|
||||
// Ensure there are no empty rows and fields are proper else filter appropriately
|
||||
// e.g. rowRDD.filter(row => row != null && row.length>0)
|
||||
val rowRDD = nyYellowTaxiRDD.map(_.split(",")).map(attr => Row(
|
||||
Timestamp.valueOf(attr(1)),
|
||||
attr(3).toInt,
|
||||
attr(4).toFloat,
|
||||
attr(16).toFloat
|
||||
)
|
||||
)
|
||||
rowRDD.take(5).foreach(println)
|
||||
//rowRDD.collect().foreach(f=>{println(f)})
|
||||
|
||||
// Apply the schema to the RDD
|
||||
val nyYellowTaxiDF = spark.createDataFrame(rowRDD, schema)
|
||||
//nyYellowTaxiDF.show()
|
||||
|
||||
// Creates a temporary view using the DataFrame
|
||||
nyYellowTaxiDF.createOrReplaceTempView("nyyellowtaxi")
|
||||
// nytaxiDF.createGlobalTempView("nytaxi")
|
||||
|
||||
//val totRides = spark.sql("select count(*) from nyyellowtaxi")
|
||||
//totRides.show()
|
||||
|
||||
return nyYellowTaxiDF;
|
||||
}
|
||||
|
||||
def runBasicCountQuery(spark: SparkSession): Unit = {
|
||||
// NyTaxi RDD
|
||||
val nyGreenTaxiRDD = spark.sparkContext.textFile("green_tripdata_2013-08.csv")
|
||||
|
||||
// Programmatic Schema with string values
|
||||
val schemaString = "VendorID Store_and_fwd_flag"
|
||||
val fields = schemaString.split(" ").map(fieldName => StructField(fieldName, StringType, nullable = true))
|
||||
val schema = StructType(fields)
|
||||
|
||||
// Convert records of the RDD (people) to Rows
|
||||
val rowRDD = nyGreenTaxiRDD.map(_.split(",")).map(attributes => Row(attributes(0), attributes(3).trim))
|
||||
// Apply the schema to the RDD
|
||||
val nyGreenTaxiDF = spark.createDataFrame(rowRDD, schema)
|
||||
|
||||
// Creates a temporary view using the DataFrame
|
||||
nyGreenTaxiDF.createOrReplaceTempView("nygreentaxi")
|
||||
// nyGreenTaxiDF.createGlobalTempView("nygreentaxi")
|
||||
|
||||
// Run the query
|
||||
val totRides = spark.sql("select count(*) from nygreentaxi")
|
||||
|
||||
// Show results
|
||||
totRides.show()
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,676 @@
|
|||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Taxi Queries on a Single DF from CSV"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Starting Spark application\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<table>\n",
|
||||
"<tr><th>ID</th><th>YARN Application ID</th><th>Kind</th><th>State</th><th>Spark UI</th><th>Driver log</th><th>Current session?</th></tr><tr><td>1</td><td>application_1586146063972_0005</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"https://pipedreamhd.azurehdinsight.net/yarnui/hn/proxy/application_1586146063972_0005/\">Link</a></td><td><a target=\"_blank\" href=\"http://wn4-pipedr.bh4dzsewnzne3doz2hewdd4akd.dx.internal.cloudapp.net:30060/node/containerlogs/container_1586146063972_0005_01_000001/livy\">Link</a></td><td>✔</td></tr></table>"
|
||||
],
|
||||
"text/plain": [
|
||||
"<IPython.core.display.HTML object>"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"SparkSession available as 'spark'.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from pyspark.sql import SparkSession\n",
|
||||
"import pyspark.sql.functions as F"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from pyspark.sql.types import *"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"taxi_file_loc=\"wasb://yellow@pipedreamdemo/taxi_data/yellow_tripdata_2019-12.csv\"\n",
|
||||
"schema_cols = \"VendorID,tpep_pickup_datetime,tpep_dropoff_datetime,passenger_count,trip_distance,RatecodeID,store_and_fwd_flag,PULocationID,DOLocationID,payment_type,fare_amount,extra,mta_tax,tip_amount,tolls_amount,improvement_surcharge,total_amount,congestion_surcharge\".split(',')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"taxi_orig_schema = (\n",
|
||||
" StructType()\n",
|
||||
" .add(\"VendorID\", IntegerType())\n",
|
||||
" .add(\"tpep_pickup_datetime\", TimestampType())\n",
|
||||
" .add(\"tpep_dropoff_datetime\", TimestampType())\n",
|
||||
" .add(\"passenger_count\", IntegerType())\n",
|
||||
" .add(\"trip_distance\", DoubleType())\n",
|
||||
" .add(\"RatecodeID\", IntegerType())\n",
|
||||
" .add(\"store_and_fwd_flag\", StringType())\n",
|
||||
" .add(\"PULocationID\", IntegerType())\n",
|
||||
" .add(\"DOLocationID\", IntegerType())\n",
|
||||
" .add(\"payment_type\", IntegerType())\n",
|
||||
" .add(\"fare_amount\", DoubleType())\n",
|
||||
" .add(\"extra\", DoubleType())\n",
|
||||
" .add(\"mta_tax\", DoubleType())\n",
|
||||
" .add(\"tip_amount\", DoubleType())\n",
|
||||
" .add(\"tolls_amount\", DoubleType())\n",
|
||||
" .add(\"improvement_surcharge\", DoubleType())\n",
|
||||
" .add(\"total_amount\", DoubleType())\n",
|
||||
" .add(\"congestion_surcharge\", DoubleType())\n",
|
||||
")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Read it"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"An error was encountered:\n",
|
||||
"Interpreter died:\n",
|
||||
"\n",
|
||||
"/usr/hdp/current/spark2-client/python/pyspark/context.py:261: RuntimeWarning: Failed to add file [file:/usr/hdp/current/spark2-client/python/lib/pyspark.zip] speficied in 'spark.submit.pyFiles' to Python path:\n",
|
||||
" /mnt/resource/hadoop/yarn/local/usercache/livy/appcache/application_1586146063972_0005/container_1586146063972_0005_01_000001/tmp\n",
|
||||
" /mnt/resource/hadoop/yarn/local/usercache/livy/appcache/application_1586146063972_0005/spark-b02c8e03-a6a6-40a4-a364-0e86f485fb21/userFiles-6c84950d-1e7c-4433-af02-3d504a2657da\n",
|
||||
" /usr/hdp/current/spark2-client/python\n",
|
||||
" /usr/hdp/current/spark2-client/python/lib/py4j-0.10.7-src.zip\n",
|
||||
" /mnt/resource/hadoop/yarn/local/usercache/livy/appcache/application_1586146063972_0005/container_1586146063972_0005_01_000001/pyspark.zip\n",
|
||||
" /mnt/resource/hadoop/yarn/local/usercache/livy/appcache/application_1586146063972_0005/container_1586146063972_0005_01_000001/py4j-0.10.7-src.zip\n",
|
||||
" /usr/bin/anaconda/lib/python27.zip\n",
|
||||
" /usr/bin/anaconda/lib/python2.7\n",
|
||||
" /usr/bin/anaconda/lib/python2.7/plat-linux2\n",
|
||||
" /usr/bin/anaconda/lib/python2.7/lib-tk\n",
|
||||
" /usr/bin/anaconda/lib/python2.7/lib-old\n",
|
||||
" /usr/bin/anaconda/lib/python2.7/lib-dynload\n",
|
||||
" /usr/bin/anaconda/lib/python2.7/site-packages\n",
|
||||
" /usr/bin/anaconda/lib/python2.7/site-packages/Sphinx-1.4.6-py2.7.egg\n",
|
||||
" /var/lib/.jupyter/jupyterazure\n",
|
||||
" /usr/bin/anaconda/lib/python2.7/site-packages/setuptools-27.2.0-py2.7.egg\n",
|
||||
" RuntimeWarning)\n",
|
||||
"/usr/hdp/current/spark2-client/python/pyspark/context.py:261: RuntimeWarning: Failed to add file [file:/usr/hdp/current/spark2-client/python/lib/py4j-0.10.7-src.zip] speficied in 'spark.submit.pyFiles' to Python path:\n",
|
||||
" /mnt/resource/hadoop/yarn/local/usercache/livy/appcache/application_1586146063972_0005/container_1586146063972_0005_01_000001/tmp\n",
|
||||
" /mnt/resource/hadoop/yarn/local/usercache/livy/appcache/application_1586146063972_0005/spark-b02c8e03-a6a6-40a4-a364-0e86f485fb21/userFiles-6c84950d-1e7c-4433-af02-3d504a2657da\n",
|
||||
" /usr/hdp/current/spark2-client/python\n",
|
||||
" /usr/hdp/current/spark2-client/python/lib/py4j-0.10.7-src.zip\n",
|
||||
" /mnt/resource/hadoop/yarn/local/usercache/livy/appcache/application_1586146063972_0005/container_1586146063972_0005_01_000001/pyspark.zip\n",
|
||||
" /mnt/resource/hadoop/yarn/local/usercache/livy/appcache/application_1586146063972_0005/container_1586146063972_0005_01_000001/py4j-0.10.7-src.zip\n",
|
||||
" /usr/bin/anaconda/lib/python27.zip\n",
|
||||
" /usr/bin/anaconda/lib/python2.7\n",
|
||||
" /usr/bin/anaconda/lib/python2.7/plat-linux2\n",
|
||||
" /usr/bin/anaconda/lib/python2.7/lib-tk\n",
|
||||
" /usr/bin/anaconda/lib/python2.7/lib-old\n",
|
||||
" /usr/bin/anaconda/lib/python2.7/lib-dynload\n",
|
||||
" /usr/bin/anaconda/lib/python2.7/site-packages\n",
|
||||
" /usr/bin/anaconda/lib/python2.7/site-packages/Sphinx-1.4.6-py2.7.egg\n",
|
||||
" /var/lib/.jupyter/jupyterazure\n",
|
||||
" /usr/bin/anaconda/lib/python2.7/site-packages/setuptools-27.2.0-py2.7.egg\n",
|
||||
" RuntimeWarning)\n",
|
||||
"\n",
|
||||
"ERROR:fake_shell:execute_reply\n",
|
||||
"Traceback (most recent call last):\n",
|
||||
" File \"/mnt/resource/hadoop/yarn/local/usercache/livy/appcache/application_1586146063972_0005/container_1586146063972_0005_01_000001/tmp/1656805614047440300\", line 318, in execute_request\n",
|
||||
" result = node.execute()\n",
|
||||
" File \"/mnt/resource/hadoop/yarn/local/usercache/livy/appcache/application_1586146063972_0005/container_1586146063972_0005_01_000001/tmp/1656805614047440300\", line 233, in execute\n",
|
||||
" raise ExecutionError(sys.exc_info())\n",
|
||||
"ExecutionError\n",
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"taxi_train_df = spark.read.schema(taxi_orig_schema).csv(taxi_file_loc, header=False)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"root\n",
|
||||
" |-- VendorID: integer (nullable = true)\n",
|
||||
" |-- tpep_pickup_datetime: timestamp (nullable = true)\n",
|
||||
" |-- tpep_dropoff_datetime: timestamp (nullable = true)\n",
|
||||
" |-- passenger_count: integer (nullable = true)\n",
|
||||
" |-- trip_distance: double (nullable = true)\n",
|
||||
" |-- RatecodeID: integer (nullable = true)\n",
|
||||
" |-- store_and_fwd_flag: string (nullable = true)\n",
|
||||
" |-- PULocationID: integer (nullable = true)\n",
|
||||
" |-- DOLocationID: integer (nullable = true)\n",
|
||||
" |-- payment_type: integer (nullable = true)\n",
|
||||
" |-- fare_amount: double (nullable = true)\n",
|
||||
" |-- extra: double (nullable = true)\n",
|
||||
" |-- mta_tax: double (nullable = true)\n",
|
||||
" |-- tip_amount: double (nullable = true)\n",
|
||||
" |-- tolls_amount: double (nullable = true)\n",
|
||||
" |-- improvement_surcharge: double (nullable = true)\n",
|
||||
" |-- total_amount: double (nullable = true)\n",
|
||||
" |-- congestion_surcharge: double (nullable = true)"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"taxi_train_df.printSchema()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"+--------+--------------------+---------------------+---------------+-------------+----------+------------------+------------+------------+------------+-----------+-----+-------+----------+------------+---------------------+------------+--------------------+\n",
|
||||
"|VendorID|tpep_pickup_datetime|tpep_dropoff_datetime|passenger_count|trip_distance|RatecodeID|store_and_fwd_flag|PULocationID|DOLocationID|payment_type|fare_amount|extra|mta_tax|tip_amount|tolls_amount|improvement_surcharge|total_amount|congestion_surcharge|\n",
|
||||
"+--------+--------------------+---------------------+---------------+-------------+----------+------------------+------------+------------+------------+-----------+-----+-------+----------+------------+---------------------+------------+--------------------+\n",
|
||||
"+--------+--------------------+---------------------+---------------+-------------+----------+------------------+------------+------------+------------+-----------+-----+-------+----------+------------+---------------------+------------+--------------------+"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"taxi_train_df.show(100)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Update the schema"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Run Queries"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"taxi_train_df.createOrReplaceTempView('nyyellowtaxi')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"+-------+\n",
|
||||
"| total|\n",
|
||||
"+-------+\n",
|
||||
"|6896317|\n",
|
||||
"+-------+"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"\n",
|
||||
"count_df = spark.sql(\"select count(*) as total from nyyellowtaxi\").show()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"+---------------+----+-------+\n",
|
||||
"|passenger_count|year| total|\n",
|
||||
"+---------------+----+-------+\n",
|
||||
"| 5|2009| 6|\n",
|
||||
"| 6|2019| 147923|\n",
|
||||
"| 9|2019| 16|\n",
|
||||
"| 4|2019| 164921|\n",
|
||||
"| 8|2019| 18|\n",
|
||||
"| 5|2020| 1|\n",
|
||||
"| 4|2009| 2|\n",
|
||||
"| 6|2008| 1|\n",
|
||||
"| 4|2008| 4|\n",
|
||||
"| 1|2058| 3|\n",
|
||||
"| 5|2019| 244875|\n",
|
||||
"| 1|2009| 16|\n",
|
||||
"| null|2019| 51018|\n",
|
||||
"| 3|2019| 302987|\n",
|
||||
"| 2|2008| 6|\n",
|
||||
"| 1|2019|4783228|\n",
|
||||
"| 1|2008| 8|\n",
|
||||
"| 3|2020| 12|\n",
|
||||
"| 4|2020| 4|\n",
|
||||
"| 1|2026| 1|\n",
|
||||
"+---------------+----+-------+\n",
|
||||
"only showing top 20 rows"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"psg_count_dist = spark.sql(\"SELECT passenger_count, \\\n",
|
||||
" year(tpep_pickup_datetime) year, count(*) total FROM nyyellowtaxi GROUP BY passenger_count, year(tpep_pickup_datetime)\").show()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"avg_fare_psg_count = spark.sql(\"SELECT passenger_count, \\\n",
|
||||
" avg(total_amount) avg_fare FROM nyyellowtaxi GROUP BY passenger_count\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 13,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"+---------------+------------------+\n",
|
||||
"|passenger_count| avg_fare|\n",
|
||||
"+---------------+------------------+\n",
|
||||
"| null| 38.10385452193327|\n",
|
||||
"| 1| 19.2689940821399|\n",
|
||||
"| 6|19.365102653367924|\n",
|
||||
"| 3|20.025870953485427|\n",
|
||||
"| 5| 19.35781789152236|\n",
|
||||
"| 9| 72.15124999999999|\n",
|
||||
"| 4|20.576281717809266|\n",
|
||||
"| 8| 94.81166666666665|\n",
|
||||
"| 7| 61.69222222222221|\n",
|
||||
"| 2| 20.35205234127457|\n",
|
||||
"| 0| 18.94103797144808|\n",
|
||||
"+---------------+------------------+"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"avg_fare_psg_count.show()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 16,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"psg_year_count = spark.sql(\"SELECT passenger_count, \\\n",
|
||||
" year(tpep_pickup_datetime) trip_year, \\\n",
|
||||
" round(trip_distance) distance, \\\n",
|
||||
" count(*) trips \\\n",
|
||||
" FROM nyyellowtaxi \\\n",
|
||||
" GROUP BY passenger_count, \\\n",
|
||||
" year(tpep_pickup_datetime), round(trip_distance) ORDER BY trip_year, trips desc\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 18,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"+---------------+---------+--------+-----+\n",
|
||||
"|passenger_count|trip_year|distance|trips|\n",
|
||||
"+---------------+---------+--------+-----+\n",
|
||||
"| 1| 2008| 1.0| 3|\n",
|
||||
"| 4| 2008| 1.0| 2|\n",
|
||||
"| 3| 2008| 1.0| 2|\n",
|
||||
"| 3| 2008| 3.0| 2|\n",
|
||||
"| 1| 2008| 0.0| 2|\n",
|
||||
"| 2| 2008| 1.0| 2|\n",
|
||||
"| 3| 2008| 0.0| 1|\n",
|
||||
"| 1| 2008| 10.0| 1|\n",
|
||||
"| 3| 2008| 4.0| 1|\n",
|
||||
"| 2| 2008| 8.0| 1|\n",
|
||||
"| 1| 2008| 12.0| 1|\n",
|
||||
"| 6| 2008| 18.0| 1|\n",
|
||||
"| 5| 2008| 0.0| 1|\n",
|
||||
"| 2| 2008| 3.0| 1|\n",
|
||||
"| 2| 2008| 0.0| 1|\n",
|
||||
"| 2| 2008| 17.0| 1|\n",
|
||||
"| 5| 2008| 1.0| 1|\n",
|
||||
"| 4| 2008| 2.0| 1|\n",
|
||||
"| 5| 2008| 2.0| 1|\n",
|
||||
"| 1| 2008| 8.0| 1|\n",
|
||||
"+---------------+---------+--------+-----+\n",
|
||||
"only showing top 20 rows"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"psg_year_count.show()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "PySpark",
|
||||
"language": "",
|
||||
"name": "pysparkkernel"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "python",
|
||||
"version": 3
|
||||
},
|
||||
"mimetype": "text/x-python",
|
||||
"name": "pyspark",
|
||||
"pygments_lexer": "python3"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 4
|
||||
}
|
|
@ -0,0 +1,280 @@
|
|||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Taxi Queries on a Structured Stream\n",
|
||||
"\n",
|
||||
"We have 106GB of NYC Yellow Taxi data that has been cleaned and normalized based on the [published schema](https://www1.nyc.gov/assets/tlc/downloads/pdf/data_dictionary_trip_records_yellow.pdf)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from pyspark.sql import SparkSession\n",
|
||||
"import pyspark.sql.functions as F\n",
|
||||
"from pyspark.sql.types import *\n",
|
||||
"import os"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"\n",
|
||||
" <div>\n",
|
||||
" <p><b>SparkSession - hive</b></p>\n",
|
||||
" \n",
|
||||
" <div>\n",
|
||||
" <p><b>SparkContext</b></p>\n",
|
||||
"\n",
|
||||
" <p><a href=\"http://localsparkeast.internal.cloudapp.net:4041\">Spark UI</a></p>\n",
|
||||
"\n",
|
||||
" <dl>\n",
|
||||
" <dt>Version</dt>\n",
|
||||
" <dd><code>v2.4.4</code></dd>\n",
|
||||
" <dt>Master</dt>\n",
|
||||
" <dd><code>local[*]</code></dd>\n",
|
||||
" <dt>AppName</dt>\n",
|
||||
" <dd><code>pyspark-shell</code></dd>\n",
|
||||
" </dl>\n",
|
||||
" </div>\n",
|
||||
" \n",
|
||||
" </div>\n",
|
||||
" "
|
||||
],
|
||||
"text/plain": [
|
||||
"<pyspark.sql.session.SparkSession at 0x7fead4b4e2d0>"
|
||||
]
|
||||
},
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"spark = SparkSession.builder.appName(\"readTaxi\").getOrCreate()\n",
|
||||
"spark"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"taxi_file_dir=os.path.expanduser(\"~/data/taxi_data_cleaned_18_standard/\")\n",
|
||||
"bad_records_path=os.path.expanduser(\"~/data/badrecords\")\n",
|
||||
"taxi_output_dir=os.path.expanduser(\"~/data/query_output\")\n",
|
||||
"checkpoint_loc=os.path.expanduser(\"~/data/checkpoints\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Define schema ahead of time for structured streaming"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"taxi_orig_schema = (\n",
|
||||
" StructType()\n",
|
||||
" .add(\"VendorID\", IntegerType())\n",
|
||||
" .add(\"tpep_pickup_datetime\", TimestampType())\n",
|
||||
" .add(\"tpep_dropoff_datetime\", TimestampType())\n",
|
||||
" .add(\"passenger_count\", IntegerType())\n",
|
||||
" .add(\"trip_distance\", DoubleType())\n",
|
||||
" .add(\"RatecodeID\", IntegerType())\n",
|
||||
" .add(\"store_and_fwd_flag\", StringType())\n",
|
||||
" .add(\"PULocationID\", IntegerType())\n",
|
||||
" .add(\"DOLocationID\", IntegerType())\n",
|
||||
" .add(\"payment_type\", IntegerType())\n",
|
||||
" .add(\"fare_amount\", DoubleType())\n",
|
||||
" .add(\"extra\", DoubleType())\n",
|
||||
" .add(\"mta_tax\", DoubleType())\n",
|
||||
" .add(\"tip_amount\", DoubleType())\n",
|
||||
" .add(\"tolls_amount\", DoubleType())\n",
|
||||
" .add(\"improvement_surcharge\", DoubleType())\n",
|
||||
" .add(\"total_amount\", DoubleType())\n",
|
||||
" .add(\"congestion_surcharge\", DoubleType())\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Define Stream Source\n",
|
||||
"\n",
|
||||
"Also get the SQL table reference for the dataframe"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"taxi_input_df = (\n",
|
||||
" spark\n",
|
||||
" .readStream\n",
|
||||
" .schema(taxi_orig_schema)\n",
|
||||
" .option(\"header\", \"false\")\n",
|
||||
" .option(\"maxFilesPerTrigger\", 1)\n",
|
||||
" .option(\"badRecordsPath\", bad_records_path)\n",
|
||||
" .csv(taxi_file_dir) #the source\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"#Create SQL table out of it\n",
|
||||
"taxi_input_df.createOrReplaceTempView('nyyellowtaxi')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Queries\n",
|
||||
"\n",
|
||||
"```sql\n",
|
||||
"1. select passenger_count, year(tpep_pickup_datetime) year from nyyellowtaxi\n",
|
||||
"2. select passenger_count, avg(total_amount) as avg_amount from nyyellowtaxi group by passenger_count\n",
|
||||
"3. select passenger_count, year(tpep_pickup_datetime) year, count(*) total \n",
|
||||
" from nyyellowtaxi group by passenger_count, year\n",
|
||||
"4. select passenger_count, year(tpep_pickup_datetime) as year, \n",
|
||||
" cast(trip_distance as int) as distance, \n",
|
||||
" count(*) as total \n",
|
||||
" from nyyellowtaxi\n",
|
||||
" group by passenger_count, year, distance\n",
|
||||
" order by year, total desc\n",
|
||||
"```\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"queries = [\"select passenger_count, year(tpep_pickup_datetime) year from nyyellowtaxi\",\n",
|
||||
" \"select passenger_count, avg(total_amount) as avg_amount from nyyellowtaxi group by passenger_count\",\n",
|
||||
" \"select passenger_count, year(tpep_pickup_datetime) year, count(*) total FROM $tableName group by passenger_count, year\",\n",
|
||||
" \"\"\"select passenger_count, year(tpep_pickup_datetime) as year, \n",
|
||||
" cast(trip_distance as int) as distance, \n",
|
||||
" count(*) as total \n",
|
||||
" from $tableName\n",
|
||||
" group by passenger_count, year, distance\n",
|
||||
" order by year, total desMc\"\"\"\n",
|
||||
" ]\n",
|
||||
"\n",
|
||||
"query_names = [\"q{}\".format(i) for i in range(1, len(queries))]\n",
|
||||
"out_dirs = [os.path.join(taxi_output_dir, n) for n in query_names]\n",
|
||||
"checkpoint_locs = [os.path.join(checkpoint_loc, n) for n in query_names]\n",
|
||||
"name_query_outdir = list(zip(query_names, queries, out_dirs, checkpoint_locs))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def start_query(query_desc):\n",
|
||||
" query_name, query_str, output_dir, checkpoint_loc = query_desc\n",
|
||||
" q = (\n",
|
||||
" spark.sql(query_str)\n",
|
||||
" .writeStream\n",
|
||||
" .outputMode(\"append\")\n",
|
||||
" .format(\"csv\")\n",
|
||||
" .option(\"path\", output_dir)\n",
|
||||
" .option(\"startingOffsets\", \"earliest\")\n",
|
||||
" .option(\"checkpointLocation\", checkpoint_loc)\n",
|
||||
" .queryName(query_name)\n",
|
||||
" .trigger(processingTime='1 seconds') \n",
|
||||
" .start()\n",
|
||||
" )\n",
|
||||
" return q"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"q1 = start_query(name_query_outdir[0])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"{'message': 'Processing new data',\n",
|
||||
" 'isDataAvailable': True,\n",
|
||||
" 'isTriggerActive': True}"
|
||||
]
|
||||
},
|
||||
"execution_count": 12,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"q1.status"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 13,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"q1.stop()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3.7 - Spark (local)",
|
||||
"language": "python",
|
||||
"name": "spark-3-python"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.7.6"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 4
|
||||
}
|
|
@ -0,0 +1,591 @@
|
|||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Taxi Queries on a Single DF from CSV"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from pyspark.sql import SparkSession\n",
|
||||
"import pyspark.sql.functions as F"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from pyspark.sql.types import *"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"taxi_file_loc=\"abfs:///taxi_data/yellow_tripdata_2019-12.csv\"\n",
|
||||
"taxi_file_dir=\"/taxi_data/\"\n",
|
||||
"taxi_output_dir=\"/taxi_data_year_count\"\n",
|
||||
"schema_cols = \"VendorID,tpep_pickup_datetime,tpep_dropoff_datetime,passenger_count,trip_distance,RatecodeID,store_and_fwd_flag,PULocationID,DOLocationID,payment_type,fare_amount,extra,mta_tax,tip_amount,tolls_amount,improvement_surcharge,total_amount,congestion_surcharge\".split(',')\n",
|
||||
"bad_recordset_path=\"/taxi_error\"\n",
|
||||
"checkpoint_loc=\"/checkpoints\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Define schema ahead of time for perf"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"taxi_orig_schema = (\n",
|
||||
" StructType()\n",
|
||||
" .add(\"VendorID\", IntegerType())\n",
|
||||
" .add(\"tpep_pickup_datetime\", TimestampType())\n",
|
||||
" .add(\"tpep_dropoff_datetime\", TimestampType())\n",
|
||||
" .add(\"passenger_count\", IntegerType())\n",
|
||||
" .add(\"trip_distance\", DoubleType())\n",
|
||||
" .add(\"RatecodeID\", IntegerType())\n",
|
||||
" .add(\"store_and_fwd_flag\", StringType())\n",
|
||||
" .add(\"PULocationID\", IntegerType())\n",
|
||||
" .add(\"DOLocationID\", IntegerType())\n",
|
||||
" .add(\"payment_type\", IntegerType())\n",
|
||||
" .add(\"fare_amount\", DoubleType())\n",
|
||||
" .add(\"extra\", DoubleType())\n",
|
||||
" .add(\"mta_tax\", DoubleType())\n",
|
||||
" .add(\"tip_amount\", DoubleType())\n",
|
||||
" .add(\"tolls_amount\", DoubleType())\n",
|
||||
" .add(\"improvement_surcharge\", DoubleType())\n",
|
||||
" .add(\"total_amount\", DoubleType())\n",
|
||||
" .add(\"congestion_surcharge\", DoubleType())\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"taxi_clean_chema = (\n",
|
||||
" StructType()\n",
|
||||
" .add(\"tpep_pickup_datetime\", TimestampType())\n",
|
||||
" .add(\"tpep_dropoff_datetime\", TimestampType())\n",
|
||||
" .add(\"passenger_count\", IntegerType())\n",
|
||||
" .add(\"trip_distance\", DoubleType())\n",
|
||||
" .add(\"payment_type\", IntegerType())\n",
|
||||
" .add(\"fare_amount\", DoubleType())\n",
|
||||
" .add(\"extra\", DoubleType())\n",
|
||||
" .add(\"mta_tax\", DoubleType())\n",
|
||||
" .add(\"tip_amount\", DoubleType())\n",
|
||||
" .add(\"tolls_amount\", DoubleType())\n",
|
||||
" .add(\"improvement_surcharge\", DoubleType())\n",
|
||||
" .add(\"total_amount\", DoubleType())\n",
|
||||
" .add(\"congestion_surcharge\", DoubleType())\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Define Input DF"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"taxi_input_df = (\n",
|
||||
" spark\n",
|
||||
" .readStream\n",
|
||||
" .option(\"maxFilesPerTrigger\", 1)\n",
|
||||
" .schema(taxi_orig_schema)\n",
|
||||
" .option(\"badRecordsPath\", bad_recordset_path) #any bad records will go here\n",
|
||||
" .option(\"header\", \"false\")\n",
|
||||
" .csv(taxi_file_dir) #the source\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"query= (\n",
|
||||
" taxi_input_df\n",
|
||||
" .writeStream\n",
|
||||
" .format(\"parquet\")\n",
|
||||
" .option(\"path\", taxi_output_dir)\n",
|
||||
" .option(\"checkpointLocation\", checkpoint_loc)\n",
|
||||
" .outputMode(\"append\")\n",
|
||||
" .option(\"startingOffsets\", \"earliest\")\n",
|
||||
" .queryName(\"taxi\")\n",
|
||||
"# .trigger(processingTime='5 seconds')\n",
|
||||
" .start()\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"query.stop()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Define Output DF (query)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Read it"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Run Queries"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"taxi_input_df.createOrReplaceTempView('nyyellowtaxi')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import uuid\n",
|
||||
"\n",
|
||||
"def write_batch_df(batch_df, epoch_id):\n",
|
||||
" try:\n",
|
||||
" batch_df.write.csv('{}/taxi_{}'.format(taxi_output_dir, epoch_id), mode='ignore')\n",
|
||||
" except:\n",
|
||||
" pass"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"q0 = (\n",
|
||||
" spark.sql(\"SELECT passenger_count, \\\n",
|
||||
" year(tpep_pickup_datetime) year, count(*) total FROM nyyellowtaxi GROUP BY passenger_count, year(tpep_pickup_datetime)\")\n",
|
||||
" .writeStream\n",
|
||||
" .outputMode(\"complete\")\n",
|
||||
" .foreachBatch(write_batch_df)\n",
|
||||
" .option(\"startingOffsets\", \"earliest\")\n",
|
||||
" .queryName(\"year_count\")\n",
|
||||
" .trigger(processingTime='20 seconds') \n",
|
||||
" .start()\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"metadata": {
|
||||
"scrolled": false
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"{u'message': u'Processing new data', u'isTriggerActive': True, u'isDataAvailable': True}"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"q0.status"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"amount = (\n",
|
||||
"taxi_input_df\n",
|
||||
".filter(F.col(\"fare_amount\") > 11.0)\n",
|
||||
".writeStream\n",
|
||||
".format(\"memory\")\n",
|
||||
".queryName(\"query4\")\n",
|
||||
".outputMode(\"append\")\n",
|
||||
".start() \n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"{u'message': u'Processing new data', u'isTriggerActive': True, u'isDataAvailable': True}"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"amount.status"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "de48abab00f64efbbdde22e22121809b",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"%%sql\n",
|
||||
"select * from query4"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 70,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"avg_fare_psg_count = spark.sql(\"SELECT passenger_count, \\\n",
|
||||
" avg(total_amount) avg_fare FROM nyyellowtaxi GROUP BY passenger_count\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 71,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"DataFrame[passenger_count: int, avg_fare: double]"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"avg_fare_psg_count"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"psg_year_count = spark.sql(\"SELECT passenger_count, \\\n",
|
||||
" year(tpep_pickup_datetime) trip_year, \\\n",
|
||||
" round(trip_distance) distance, \\\n",
|
||||
" count(*) trips \\\n",
|
||||
" FROM nyyellowtaxi \\\n",
|
||||
" GROUP BY passenger_count, \\\n",
|
||||
" year(tpep_pickup_datetime), round(trip_distance) ORDER BY trip_year, trips desc\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"psg_year_count.show()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3.7 - Spark (local)",
|
||||
"language": "python",
|
||||
"name": "spark-3-python"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.7.6"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 4
|
||||
}
|
|
@ -0,0 +1,246 @@
|
|||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 98,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from faker import Faker\n",
|
||||
"import random as rnd\n",
|
||||
"import os, argparse, logging\n",
|
||||
"import pandas as pd\n",
|
||||
"\n",
|
||||
"logging.basicConfig(format='%(asctime)s %(levelname)-10s %(message)s', datefmt=\"%Y-%m-%d-%H-%M-%S\",\n",
|
||||
" level=logging.INFO)\n",
|
||||
"\n",
|
||||
"def parse_args():\n",
|
||||
" ap = argparse.ArgumentParser()\n",
|
||||
" ap.add_argument(\"-o\", \"--outpath\", default=\".\", help=\"Output for csv files\")\n",
|
||||
" ap.add_argument(\"-n\", \"--n-files\", default=5, type=int, help=\"number of files to generate\")\n",
|
||||
" ap.add_argument(\"-s\", \"--file-size\", default=10 * 1024 * 1024, type=int, help=\"file size\")\n",
|
||||
" return ap.parse_args()\n",
|
||||
"\n",
|
||||
"columns = (\n",
|
||||
" [\"Text01\", \"Text02\", \"Text03\", \"Text04\", \"Text05\", \"Text06\", \"IntWritable01\", \"Text07\", \"Text08\"] +\n",
|
||||
" [\"Text09\", \"Text10\", \"Text11\", \"Text12\", \"Text13\", \"LongWritable01\", \"LongWritable02\", \"LongWritable03\"] +\n",
|
||||
" [\"LongWritable04\", \"LongWritable05\", \"LongWritable06\", \"Text14\", \"Text15\", \"IntWritable02\", \"Text16\", \"Text17\", \"Text18\", \"Text19\", \"Text20\",] +\n",
|
||||
" [\"IntWritable03\", \"IntWritable04\", \"Text21\", \"IntWritable05\", \"Text22\", \"Text23\", \"Text24\", \"Text25\", \"Text26\", \"IntWritable06\"] +\n",
|
||||
" [\"IntWritable07\", \"Text27\", \"IntWritable08\", \"Text28\", \"IntWritable09\", \"IntWritable10\", \"IntWritable11\", \"Text29\", \"Text30\", \"Text31\"]\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"fake = Faker()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 99,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def get_date():\n",
|
||||
" return fake.date(pattern='%m:%d:%y %H:%M:%S:%f')[:-3]\n",
|
||||
"\n",
|
||||
"def get_dec(a, b):\n",
|
||||
" return str(fake.pydecimal(left_digits=rnd.randint(a, b), right_digits=0))[:-2].replace('-', '')\n",
|
||||
"\n",
|
||||
"def str_exact_chars(n):\n",
|
||||
" return fake.pystr(min_chars=n, max_chars=n)\n",
|
||||
"\n",
|
||||
"def generate_record():\n",
|
||||
" Text01 = get_date()\n",
|
||||
" Text02 = get_date()\n",
|
||||
" Text03 = get_dec(1, 15)\n",
|
||||
" Text04 = str_exact_chars(3)\n",
|
||||
" Text05 = get_dec(1, 15)\n",
|
||||
" Text06 = fake.pystr_format(string_format='###-###-###-###-##{{random_int}}')[:-4]\n",
|
||||
" IntWriteable01 = get_dec(1, 1)\n",
|
||||
" Text07 = fake.pystr_format(string_format='###-###-###-###-####{{random_int}}')[:-2]\n",
|
||||
" Text08 = fake.ipv6()\n",
|
||||
" Text09 = fake.ipv6()\n",
|
||||
" Text10 = ''\n",
|
||||
" Text11 = get_dec(5, 5)\n",
|
||||
" Text12 = get_dec(5, 5)\n",
|
||||
" Text13 = get_dec(2, 2)\n",
|
||||
" LongWritable01 = get_dec(8, 12)\n",
|
||||
" LongWritable02 = get_dec(8, 12)\n",
|
||||
" LongWritable03 = get_dec(8, 12)\n",
|
||||
" LongWritable04 = get_dec(8, 12)\n",
|
||||
" LongWritable05 = get_dec(8, 12)\n",
|
||||
" LongWritable06 = get_dec(8, 12)\n",
|
||||
" Text14 = str_exact_chars(10)\n",
|
||||
" Text15 = str_exact_chars(10)\n",
|
||||
" IntWriteable02 = fake.pyint()\n",
|
||||
" Text16 = str_exact_chars(12)\n",
|
||||
" Text17= fake.pystr_format(string_format='##{{random_int}}')\n",
|
||||
" Text18 = fake.ipv4()\n",
|
||||
" Text19 = str_exact_chars(3)\n",
|
||||
" Text20 = fake.pystr_format(string_format=\"???_?_??_???_????_?\")\n",
|
||||
" IntWriteable03 = fake.pyint() \n",
|
||||
" IntWriteable04 = fake.pyint() \n",
|
||||
" Text21 = fake.pystr_format(string_format=\"#???###????###??\")\n",
|
||||
" IntWriteable05 = fake.pyint() \n",
|
||||
" Text22 = get_dec(1, 2)\n",
|
||||
" Text23 = \"\"\n",
|
||||
" Text24 = \"\"\n",
|
||||
" Text25 = fake.pystr_format(string_format=\"##|###|#####|###|##|####\")\n",
|
||||
" Text26 = fake.pystr_format(string_format=\"##:#######:###:####\")\n",
|
||||
" IntWriteable06 = fake.pyint()\n",
|
||||
" IntWriteable07 = fake.pyint()\n",
|
||||
" Text27 = fake.ipv6()\n",
|
||||
" IntWriteable08 = fake.pyint()\n",
|
||||
" Text28 = fake.ipv6()\n",
|
||||
" IntWriteable09 = fake.pyint()\n",
|
||||
" IntWriteable10 = fake.pyint()\n",
|
||||
" IntWriteable11 = fake.pyint()\n",
|
||||
" Text29 = \"\"\n",
|
||||
" Text30 = str_exact_chars(24)\n",
|
||||
" Text31 = \"\"\n",
|
||||
"\n",
|
||||
" data = ([Text01, Text02, Text03, Text04, Text05, Text06, IntWriteable01, Text07, Text08, Text09, Text10] + \n",
|
||||
" [Text11, Text12, Text13, LongWritable01, LongWritable02, LongWritable03, LongWritable04, LongWritable05, LongWritable06] +\n",
|
||||
" [Text14, Text15, IntWriteable02, Text16, Text17, Text18, Text19, Text20, IntWriteable03, IntWriteable04, Text21] +\n",
|
||||
" [IntWriteable05, Text22, Text23, Text24, Text25, Text26, IntWriteable06, IntWriteable07, Text27, IntWriteable08] +\n",
|
||||
" [Text28, IntWriteable09, IntWriteable10, IntWriteable11, Text29, Text30, Text31]\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" return data\n",
|
||||
"\n",
|
||||
"def generate_fake_data(out_path, file_size=1024 * 1024 * 10):\n",
|
||||
" size = 0\n",
|
||||
" data_dicts = []\n",
|
||||
"\n",
|
||||
" while size < file_size:\n",
|
||||
"\n",
|
||||
" data = generate_record()\n",
|
||||
" size = sum([len(str(x)) for x in data])\n",
|
||||
" data_dicts.append(dict(zip(columns, data)))\n",
|
||||
"\n",
|
||||
" df = pd.DataFrame(data_dicts)\n",
|
||||
" df.to_csv(out_path)\n",
|
||||
" logging.info(f\"Generated: {out_path}\")\n",
|
||||
"\n",
|
||||
"def generate_files(out_path, n_files = 5, file_size=1024 * 1024 * 10):\n",
|
||||
"\n",
|
||||
" file_path = os.path.join(out_path, fake.pystr_format(string_format=\"??????????.csv\"))\n",
|
||||
"\n",
|
||||
" for _ in range(n_files):\n",
|
||||
" generate_fake_data(file_path, file_size)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 100,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def gzip_file(file_path):\n",
|
||||
"\n",
|
||||
" base = os.path.splitext(file_path)[0]\n",
|
||||
" dst_path = f\"{base}.gz\"\n",
|
||||
"\n",
|
||||
" with open(file_path, \"r\") as fd:\n",
|
||||
" with gzip.open(dst_path, \"wb\") as gzfd:\n",
|
||||
" shutil.copyfileobj(fd, gzfd)\n",
|
||||
"\n",
|
||||
" os.remove(file_path)\n",
|
||||
" logging.info(f\"Compressed: {dst_path}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 94,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"['04:04:06 06:17:23:000',\n",
|
||||
" '09:09:07 04:47:56:000',\n",
|
||||
" '237503428635467',\n",
|
||||
" 'VZS',\n",
|
||||
" '93163749932',\n",
|
||||
" '069-678-346-387-77',\n",
|
||||
" '2',\n",
|
||||
" '388-630-450-251-613450',\n",
|
||||
" '68af:b5ce:3e7f:1565:3c6:dc2a:5417:2447',\n",
|
||||
" 'ca82:f7ed:b38d:d4f2:cf1f:f172:a761:e223',\n",
|
||||
" '',\n",
|
||||
" '63446',\n",
|
||||
" '44989',\n",
|
||||
" '23',\n",
|
||||
" '97680902',\n",
|
||||
" '170586192',\n",
|
||||
" '78100466769',\n",
|
||||
" '4131020837',\n",
|
||||
" '622047749685',\n",
|
||||
" '229398760573',\n",
|
||||
" 'hXPMiqVClQ',\n",
|
||||
" 'vdFhRsuEnf',\n",
|
||||
" 5676,\n",
|
||||
" 'dmmGfDPnQTbW',\n",
|
||||
" '986369',\n",
|
||||
" '198.4.249.177',\n",
|
||||
" 'cts',\n",
|
||||
" 'Bpb_A_HH_mGq_YaYU_u',\n",
|
||||
" 9605,\n",
|
||||
" 3593,\n",
|
||||
" '7gvq572Bwqv923pS',\n",
|
||||
" 7436,\n",
|
||||
" '11',\n",
|
||||
" '',\n",
|
||||
" '',\n",
|
||||
" '29|862|65115|481|49|5723',\n",
|
||||
" '80:9163731:799:7541',\n",
|
||||
" 7914,\n",
|
||||
" 4187,\n",
|
||||
" '175c:6e04:cb52:875e:8ace:d0f4:69de:105a',\n",
|
||||
" 5298,\n",
|
||||
" 'ac79:96f:415d:a84:7b01:eb18:8541:fe01',\n",
|
||||
" 4469,\n",
|
||||
" 7440,\n",
|
||||
" 3138,\n",
|
||||
" '',\n",
|
||||
" 'URWJjesgxDqJTeQweycMgcyq',\n",
|
||||
" '']"
|
||||
]
|
||||
},
|
||||
"execution_count": 94,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.6.10"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 4
|
||||
}
|
100
README.md
|
@ -1,64 +1,76 @@
|
|||
---
|
||||
page_type: sample
|
||||
languages:
|
||||
- csharp
|
||||
products:
|
||||
- dotnet
|
||||
description: "Add 150 character max description"
|
||||
urlFragment: "update-this-to-unique-url-stub"
|
||||
---
|
||||
# Asessing CPU vs FPGA Performance Using Spark
|
||||
|
||||
# Official Microsoft Sample
|
||||
We benchmark performance of a simple query using [NYC Taxi Dataset](https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page).
|
||||
|
||||
<!--
|
||||
Guidelines on README format: https://review.docs.microsoft.com/help/onboard/admin/samples/concepts/readme-template?branch=master
|
||||
![perf web app](docs/images/demo.png)
|
||||
|
||||
Guidance on onboarding samples to docs.microsoft.com/samples: https://review.docs.microsoft.com/help/onboard/admin/samples/process/onboarding?branch=master
|
||||
## Downloading and Normalizing Data
|
||||
|
||||
Taxonomies for products and languages: https://review.docs.microsoft.com/new-hope/information-architecture/metadata/taxonomies?branch=master
|
||||
-->
|
||||
We use a subset of the Yellow Trip Data consisting of files with 18 columns. We further normalize the data in these files to conform to the schema published on the NYC Taxi Dataset site for [Yellow Taxi trips](https://www1.nyc.gov/assets/tlc/downloads/pdf/data_dictionary_trip_records_yellow.pdf).
|
||||
|
||||
Give a short description for your sample here. What does it do and why is it important?
|
||||
To download the data locally:
|
||||
|
||||
## Contents
|
||||
1. Clone this repo
|
||||
1. Clone [nyc-taxi-data](https://github.com/toddwschneider/nyc-taxi-data) git repo
|
||||
1. Edit `download_raw_data.sh` in the root of `nyc-taxi-data` repo replacing the default `setup_files/raw_data_urls.txt` with `<this_repo>/dataset/yellow_taxi_files.csv`
|
||||
1. Run `./download_raw_data.sh` to download the dataset.
|
||||
1. Normalize the downloaded dataset by going through the `Notebooks/Standardize Schema.ipynb` in this repo. The `clean_schema` function is what does normalization.
|
||||
|
||||
Outline the file contents of the repository. It helps users navigate the codebase, build configuration and any related assets.
|
||||
Our dataset is 100 GB in size split into 82 files.
|
||||
|
||||
| File/folder | Description |
|
||||
|-------------------|--------------------------------------------|
|
||||
| `src` | Sample source code. |
|
||||
| `.gitignore` | Define what to ignore at commit time. |
|
||||
| `CHANGELOG.md` | List of changes to the sample. |
|
||||
| `CONTRIBUTING.md` | Guidelines for contributing to the sample. |
|
||||
| `README.md` | This README file. |
|
||||
| `LICENSE` | The license for the sample. |
|
||||
## Collecting Performance Data
|
||||
|
||||
## Prerequisites
|
||||
We use local Spark configured with `local[*]` (default) and structured streaming to measure and aggregate performance on our dataset. Each streaming batch consists of a single CSV file. The profiled query is:
|
||||
|
||||
Outline the required components and tools that a user might need to have on their machine in order to run the sample. This can be anything from frameworks, SDKs, OS versions or IDE releases.
|
||||
```sql
|
||||
select payment_type, count(*) as total from nyctaxidata group by payment_type
|
||||
```
|
||||
|
||||
## Setup
|
||||
As configured above, all local CPU cores are utilized for the query.
|
||||
|
||||
Explain how to prepare the sample once the user clones or downloads the repository. The section should outline every step necessary to install dependencies and set up any settings (for example, API keys and output folders).
|
||||
### CPU
|
||||
|
||||
## Running the sample
|
||||
1. [Install Apache Spark](https://www.apache.org/dyn/closer.lua/spark/spark-2.4.5/spark-2.4.5-bin-hadoop2.7.tgz).
|
||||
1. In `<repo_root>/benchmarking/queries/benchmark_taxi.scala`, modify the following values as appropriate:
|
||||
|
||||
Outline step-by-step instructions to execute the sample and see its output. Include steps for executing the sample from the IDE, starting specific services in the Azure portal or anything related to the overall launch of the code.
|
||||
```scala
|
||||
val rootPath = s"~/data/taxi_data_cleaned_18_standard" //root of the dataset
|
||||
val magentaOutDir = s"~/data/queries_e8/$queryName/processed/results" // query results
|
||||
val checkpointLoc = s"~/data/queries_e8/$queryName/checkpoint" // checkpoint files
|
||||
val logDir = s"~/data/queries_e8/$queryName/monitor/results" // profiling results
|
||||
```
|
||||
3. Launch spark-shell with enough memory to stream the data:
|
||||
|
||||
## Key concepts
|
||||
From the root of this repo:
|
||||
|
||||
Provide users with more context on the tools and services used in the sample. Explain some of the code that is being used and how services interact with each other.
|
||||
```sh
|
||||
$ cd benchmarking/queries
|
||||
$ spark-shell --driver-memory 50G
|
||||
```
|
||||
|
||||
## Contributing
|
||||
4. Load the relevant file and launch Spark processing
|
||||
|
||||
This project welcomes contributions and suggestions. Most contributions require you to agree to a
|
||||
Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us
|
||||
the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com.
|
||||
```scala
|
||||
scala> :load benchmark_taxi.scala
|
||||
scala> Benchmark.main(1)
|
||||
```
|
||||
|
||||
When you submit a pull request, a CLA bot will automatically determine whether you need to provide
|
||||
a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions
|
||||
provided by the bot. You will only need to do this once across all repos using our CLA.
|
||||
Benchmarking results will be placed in the directories prefixed with `logDir`, so in the example above these will be:
|
||||
```
|
||||
~/data/queries_e8/q1/monitor/results_0
|
||||
~/data/queries_e8/q1/monitor/results_1
|
||||
~/data/queries_e8/q1/monitor/results_2
|
||||
etc
|
||||
```
|
||||
5. Collect the results:
|
||||
```sh
|
||||
$ cat ~/data/queries_e8/q1/monitor/results_*/*.csv > taxi_q1_profile.csv
|
||||
```
|
||||
|
||||
This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
|
||||
For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or
|
||||
contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.
|
||||
### FPGA
|
||||
|
||||
Provisioning an NP-10 machine in Azure and going through the above steps should yield the benchmarks for FPGA. For the demo we used a custom "one-off" implementation of this query to assess FPGA performance.
|
||||
|
||||
## Web UI
|
||||
|
||||
See this [README](demo/README.md) for instruction on how to visualize performance data in Web UI. Here is the [finished app](https://speeddemo.z5.web.core.windows.net/)
|
|
@ -0,0 +1,160 @@
|
|||
// *********************************************************************
|
||||
// Copyright (C) Microsoft. All rights reserved.
|
||||
// *********************************************************************
|
||||
#include <Windows.h>
|
||||
|
||||
#include <chrono>
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <sstream>
|
||||
|
||||
#include "Mison/Config.h"
|
||||
#include "Mison/CsvParser.h"
|
||||
#include "Mison/CsvReader.h"
|
||||
#include "Mison/CsvValue.h"
|
||||
#include "Mison/Field.h"
|
||||
#include "Mison/JsonParser.h"
|
||||
#include "Mison/JsonReader.h"
|
||||
#include "Mison/JsonValue.h"
|
||||
#include "Mison/Nullable.h"
|
||||
#include "Mison/Record.h"
|
||||
#include "Mison/Records.h"
|
||||
|
||||
using namespace Mison;
|
||||
|
||||
/// <summary>
|
||||
/// Extract three integer fields from TPC-H lineitem CSV file.
|
||||
/// Expected parsing throughput: ~835MB/s (Intel Haswell 3.5GHz CPU).
|
||||
/// </summary>
|
||||
void CsvParserTpchBenchmark(std::string& path) {
|
||||
std::cout << std::endl;
|
||||
std::cout << "##### Csv Parser TPC-H lineitem parsing benchmark #####" << std::endl;
|
||||
|
||||
// Extract the first, second, and fifth (integer) fields.
|
||||
Config config;
|
||||
config.csv.field_delimiter = ',';
|
||||
|
||||
|
||||
for (size_t i = 0; i < 18; i++) {
|
||||
std::vector<std::string> fields;
|
||||
|
||||
for (size_t j = 0; j <= i; j++) {
|
||||
std::stringstream ss;
|
||||
ss << "$" << j;
|
||||
fields.push_back(ss.str());
|
||||
}
|
||||
|
||||
Mison::CsvParser parser(fields, config);
|
||||
|
||||
// Preload data into memory, to exclude I/O time.
|
||||
std::unique_ptr<std::ifstream> file(new std::ifstream(path, std::ifstream::binary));
|
||||
|
||||
file->seekg(0, file->end);
|
||||
size_t length = file->tellg();
|
||||
|
||||
char* buffer = new char[length];
|
||||
file->seekg(0, file->beg);
|
||||
|
||||
// Start timer.
|
||||
auto start = std::chrono::high_resolution_clock::now();
|
||||
|
||||
file->read(buffer, length);
|
||||
|
||||
CsvRecords records = parser.Parse(buffer, length, false);
|
||||
|
||||
size_t count = 0;
|
||||
int v;
|
||||
double d;
|
||||
NullableString s;
|
||||
|
||||
for (const CsvRecord& record : records) {
|
||||
for (const CsvValue& field_value : record) {
|
||||
StringRef string;
|
||||
switch (field_value.FieldId()) {
|
||||
case 0:
|
||||
field_value.GetAsString();
|
||||
break;
|
||||
case 1:
|
||||
field_value.GetAsDateTime().Value();
|
||||
break;
|
||||
case 2:
|
||||
field_value.GetAsDateTime().Value();
|
||||
break;
|
||||
case 3:
|
||||
field_value.GetAsInt().Value();
|
||||
break;
|
||||
case 4:
|
||||
field_value.GetAsDouble().Value();
|
||||
break;
|
||||
case 5:
|
||||
field_value.GetAsDouble().Value();
|
||||
break;
|
||||
case 6:
|
||||
field_value.GetAsString().Value();
|
||||
break;
|
||||
case 7:
|
||||
field_value.GetAsString().Value();
|
||||
break;
|
||||
case 8:
|
||||
field_value.GetAsString().Value();
|
||||
break;
|
||||
case 9:
|
||||
field_value.GetAsString().Value();
|
||||
break;
|
||||
case 10:
|
||||
field_value.GetAsDouble().Value();
|
||||
break;
|
||||
case 11:
|
||||
field_value.GetAsRawText();
|
||||
break;
|
||||
case 12:
|
||||
field_value.GetAsDouble().Value();
|
||||
break;
|
||||
case 13:
|
||||
field_value.GetAsDouble().Value();
|
||||
break;
|
||||
case 14:
|
||||
field_value.GetAsDouble().Value();
|
||||
break;
|
||||
case 15:
|
||||
field_value.GetAsDouble().Value();
|
||||
break;
|
||||
case 16:
|
||||
field_value.GetAsDouble().Value();
|
||||
break;
|
||||
case 17:
|
||||
field_value.GetAsDouble().Value();
|
||||
break;
|
||||
}
|
||||
}
|
||||
count++;
|
||||
}
|
||||
|
||||
// Stop timer.
|
||||
auto end = std::chrono::high_resolution_clock::now();
|
||||
size_t time = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();
|
||||
|
||||
std::cout << "========================== Cols: " << fields.size() << std::endl;
|
||||
std::cout << "Parsed " << count << " records." << std::endl;
|
||||
std::cout << "Parsing time: " << time / 1000.0 << " ms" << std::endl;
|
||||
std::cout << "Pasrsing throughput: " << static_cast<double>(length) / time << " MB/s"
|
||||
<< std::endl;
|
||||
|
||||
delete[] buffer;
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
SetThreadAffinityMask(GetCurrentThread(), 1);
|
||||
std::vector<std::string> paths = {// R"(G:\taxi_data\yellow_tripdata_2019-12.csv)",
|
||||
R"(G:\taxi_data\yellow_tripdata_2012-03.csv)"};
|
||||
|
||||
for (auto s : paths) {
|
||||
CsvParserTpchBenchmark(s);
|
||||
// CsvReaderTpchBenchmark();
|
||||
}
|
||||
return 0;
|
||||
}
|
|
@ -0,0 +1,86 @@
|
|||
import org.apache.spark.sql.Row
|
||||
|
||||
import org.apache.spark.sql.streaming.StreamingQueryListener
|
||||
import org.apache.spark.sql.streaming.StreamingQueryListener._
|
||||
import org.apache.spark.internal.Logging
|
||||
import java.util.concurrent.locks.{Lock, ReentrantLock}
|
||||
|
||||
import spark.implicits._
|
||||
|
||||
case class Record(timestamp : String, numRows : Long, avgRowSize : Int,
|
||||
inputRowsPerSec : Double, processedRowsPerSec : Double, inputDurationSec : Double, processedDurationSec : Double)
|
||||
|
||||
// this is how you use toDF
|
||||
// 1. import spark.implicits._ from an existing spark session
|
||||
// 2. define the Record case class containing the schema
|
||||
// 3. Use in a separate object outside of the class definition
|
||||
object ConvToDf {
|
||||
def createDF(seqDF : List[Record]) = {
|
||||
seqDF.toDF.na.fill(0)
|
||||
}
|
||||
}
|
||||
|
||||
class StreamMonitor(val fileName : String, val rowSize: Int, val capacity: Int = 1) extends StreamingQueryListener() with Logging {
|
||||
|
||||
private[this] val lock : Lock = new ReentrantLock()
|
||||
private[this] var collectedRecs = List[Record]()
|
||||
private[this] var queryDone = false
|
||||
private[this] var batch = 0
|
||||
|
||||
override def onQueryProgress(queryProgress: QueryProgressEvent): Unit = {
|
||||
|
||||
val progress = queryProgress.progress
|
||||
val nr = progress.numInputRows
|
||||
|
||||
// since we are streaming the query it's never done
|
||||
// check what the next batch brings and terminate processing accordingly
|
||||
if (nr == 0 || collectedRecs.length >= capacity) {
|
||||
if (queryDone) return
|
||||
try {
|
||||
lock.lock()
|
||||
if(!queryDone) {
|
||||
queryDone = nr == 0
|
||||
|
||||
if (collectedRecs.length != 0) {
|
||||
ConvToDf.createDF(collectedRecs.reverse)
|
||||
.write
|
||||
.option("header", "false")
|
||||
.csv(s"${fileName}_$batch")
|
||||
batch += 1
|
||||
}
|
||||
collectedRecs = List[Record]()
|
||||
}
|
||||
}
|
||||
catch {
|
||||
case e: Throwable => {
|
||||
logError(s"$e")
|
||||
if (!queryDone) return
|
||||
}
|
||||
}
|
||||
finally {
|
||||
lock.unlock()
|
||||
if (queryDone) return
|
||||
}
|
||||
}
|
||||
|
||||
val ts = progress.timestamp
|
||||
val inpRowsSec = progress.inputRowsPerSecond
|
||||
val procRowsSec = progress.processedRowsPerSecond
|
||||
val inpDurationSec = nr / inpRowsSec
|
||||
val procDurationSec = nr / procRowsSec
|
||||
|
||||
try {
|
||||
lock.lock()
|
||||
collectedRecs = Record(ts, nr, rowSize, inpRowsSec, procRowsSec, inpDurationSec, procDurationSec) :: collectedRecs
|
||||
}
|
||||
catch {
|
||||
case e: Throwable => logError(s"$e")
|
||||
}
|
||||
finally {
|
||||
lock.unlock()
|
||||
}
|
||||
}
|
||||
|
||||
def onQueryStarted(event: QueryStartedEvent): Unit = ()
|
||||
def onQueryTerminated(event: QueryTerminatedEvent): Unit = ()
|
||||
}
|
|
@ -0,0 +1,44 @@
|
|||
To run, do following
|
||||
|
||||
*Install AzureXRT stuff
|
||||
cd AzureXRTPackages
|
||||
sudo apt install ./*.deb
|
||||
Reboot the machine.
|
||||
|
||||
run
|
||||
source /opt/xilinx/xrt/setup.sh
|
||||
xbutil validate
|
||||
|
||||
this should pass
|
||||
|
||||
* Install required tools using following commands
|
||||
sudo apt-get install uuid-dev
|
||||
sudo apt-get install libssl-dev
|
||||
sudo apt-get install build-essential
|
||||
sudo apt-get install gcc-8
|
||||
sudo apt-get install g++-8
|
||||
sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 60 --slave /usr/bin/g++ g++ /usr/bin/g++-8
|
||||
sudo apt install python
|
||||
|
||||
* Install cmake. Follow instructions from https://linux4one.com/how-to-install-cmake-on-ubuntu-18-04-linux/
|
||||
|
||||
* Setup environment variables (Add it in .bashrc file for convenience)
|
||||
export WFDS_BUILD_ROOT=/home/boris/fpga_demo
|
||||
export LD_LIBRARY_PATH=/home/boris/fpga_demo/cpp/external_libs/poco.1.9.2/lib
|
||||
|
||||
* Build boost and Poco
|
||||
Go to cpp/external_libs/poco.1.9.2 and run the buildall.sh script.
|
||||
Go to cpp/external_libs/boost and run the buildall.sh script.
|
||||
|
||||
* Go to build_output/global
|
||||
cmake ../../
|
||||
cd accel
|
||||
make acceltesthw
|
||||
|
||||
|
||||
* Run executable like
|
||||
go to fpga_demo
|
||||
cpp/build_output/bin/acceltesthw hw -file yellow_tripdata_2019-12.csv
|
||||
Generates vstream_<csv-file> directory with VStream output
|
||||
Use smaller csv file for now. Big ones have issues
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
import org.apache.spark.sql.functions.{_ => F}
|
||||
import org.apache.spark.sql.SparkSession
|
||||
import org.apache.spark.sql.functions._
|
||||
import org.apache.spark.sql.types._
|
||||
import org.apache.spark.sql.streaming.Trigger
|
||||
import org.apache.spark.sql.streaming.StreamingQuery
|
||||
import org.apache.spark.sql.DataFrame
|
||||
|
||||
import org.apache.log4j._
|
||||
import java.nio.file.Paths
|
||||
import org.apache.spark.SparkConf
|
||||
|
||||
:load ../listener/querylistener.scala
|
||||
:load schemas.scala
|
||||
|
||||
object BenchMagenta {
|
||||
var cols = Array("")
|
||||
|
||||
def getSchema() = Schemas.getMagentaSchema
|
||||
|
||||
def loadData(filePath: String, queryName: String) = {
|
||||
|
||||
val taxi_orig_schema = getSchema()
|
||||
|
||||
val df = spark
|
||||
.readStream
|
||||
.schema(taxi_orig_schema)
|
||||
.option("header", "false")
|
||||
.option("maxFilesPerTrigger", 1)
|
||||
.csv(filePath)
|
||||
|
||||
df.createOrReplaceTempView(queryName)
|
||||
cols = df.columns
|
||||
println(s"Created sql table from dataframe")
|
||||
}
|
||||
|
||||
/** Our main function where the action happens */
|
||||
def runFile(magentaOutDir : String, checkpointLoc: String, tableName: String) = {
|
||||
|
||||
spark.sql(s"select * from $tableName")
|
||||
.writeStream
|
||||
.format("parquet")
|
||||
.outputMode("append")
|
||||
.option("path", magentaOutDir)
|
||||
.option("startingOffsets", "earliest")
|
||||
.option("checkpointLocation", checkpointLoc)
|
||||
.queryName("magenta_out")
|
||||
.start()
|
||||
}
|
||||
|
||||
def main() = {
|
||||
val rootPath = "~/yellow/magenta/raw"
|
||||
val magentaOutDir = "~/yellow/magenta/processed"
|
||||
val checkpointLoc = "~/yellow/magenta/checkpoint"
|
||||
val logDir = "~/yellow/magenta/monitor/stats"
|
||||
|
||||
println(s"spark.driver.memory: ${spark.conf.get("spark.driver.memory")}")
|
||||
println(s"Running query")
|
||||
|
||||
spark.streams.addListener(new StreamMonitor(logDir, 520, 5))
|
||||
|
||||
// start stream-reading
|
||||
loadData(rootPath, "magenta")
|
||||
|
||||
// start stsream-writing
|
||||
runFile(magentaOutDir, checkpointLoc, "magenta")
|
||||
}
|
||||
}
|
|
@ -0,0 +1,83 @@
|
|||
import org.apache.spark.sql.functions.{_ => F}
|
||||
import org.apache.spark.sql.SparkSession
|
||||
import org.apache.spark.sql.functions._
|
||||
import org.apache.spark.sql.types._
|
||||
import org.apache.spark.sql.DataFrame
|
||||
import org.apache.log4j._
|
||||
import java.nio.file.Paths
|
||||
import org.apache.spark.SparkConf
|
||||
import org.apache.spark.sql.streaming.Trigger
|
||||
|
||||
:load schemas.scala
|
||||
:load ../listener/querylistener.scala
|
||||
|
||||
object Benchmark {
|
||||
|
||||
def getSchema() = Schemas.getTaxiSchema
|
||||
|
||||
def loadData(filePath: String, queryName: String) = {
|
||||
|
||||
val taxi_orig_schema = getSchema()
|
||||
|
||||
val df = spark
|
||||
.readStream
|
||||
.schema(taxi_orig_schema)
|
||||
.option("header", "false")
|
||||
.option("maxFilesPerTrigger", 1)
|
||||
.csv(filePath)
|
||||
|
||||
df.createOrReplaceTempView(queryName)
|
||||
println(s"Created sql table from dataframe")
|
||||
}
|
||||
|
||||
/** Our main function where the action happens */
|
||||
def runFile(outDir: String, checkpointLoc: String, queryStr: String, tableName: String) = {
|
||||
|
||||
spark.sql(queryStr)
|
||||
.writeStream
|
||||
.foreachBatch((df : DataFrame, epoch : Long) => {
|
||||
df.write
|
||||
.csv(s"${outDir}_$epoch")
|
||||
})
|
||||
.outputMode("complete")
|
||||
.option("startingOffsets", "earliest")
|
||||
.option("checkpointLocation", checkpointLoc)
|
||||
.trigger(Trigger.ProcessingTime("1 second"))
|
||||
.queryName("magenta_out")
|
||||
.start()
|
||||
|
||||
}
|
||||
|
||||
def main(queryIdx: Int) = {
|
||||
val tableName = "nyyellowtaxi"
|
||||
|
||||
val queryList = List(s"select count(*) as total from $tableName",
|
||||
s"select payment_type, count(*) as total from $tableName group by payment_type",
|
||||
s"select passenger_count, avg(total_amount) as avg_amount from $tableName group by passenger_count",
|
||||
s"select passenger_count, year(tpep_pickup_datetime) year, count(*) total FROM $tableName group by passenger_count, year",
|
||||
s"""select passenger_count, year(tpep_pickup_datetime) as year,
|
||||
cast(trip_distance as int) as distance,
|
||||
count(*) as total
|
||||
from $tableName
|
||||
group by passenger_count, year, distance
|
||||
order by year, total desc
|
||||
"""
|
||||
)
|
||||
val queryName = s"q$queryIdx"
|
||||
val rootPath = s"~/data/taxi_data_cleaned_18_standard" //root of the dataset
|
||||
val magentaOutDir = s"~/data/queries_e8/$queryName/processed/results" // query results
|
||||
val checkpointLoc = s"~/data/queries_e8/$queryName/checkpoint" // checkpoint files
|
||||
val logDir = s"~/data/queries_e8/$queryName/monitor/results" // profiling results
|
||||
|
||||
println(s"spark.driver.memory: ${spark.conf.get("spark.driver.memory")}")
|
||||
println(s"Running query")
|
||||
|
||||
spark.streams.addListener(new StreamMonitor(logDir, 98, 5))
|
||||
|
||||
// start stream-reading
|
||||
loadData(rootPath, tableName)
|
||||
|
||||
// start stsream-writing
|
||||
runFile(magentaOutDir, checkpointLoc, queryList(queryIdx), tableName)
|
||||
}
|
||||
}
|
|
@ -0,0 +1,94 @@
|
|||
import org.apache.spark.sql.functions.{_ => F}
|
||||
import org.apache.spark.sql.SparkSession
|
||||
import org.apache.spark.sql.functions._
|
||||
import org.apache.spark.sql.types._
|
||||
import org.apache.spark.sql.DataFrame
|
||||
import org.apache.log4j._
|
||||
import org.apache.spark.SparkConf
|
||||
import org.apache.spark.sql.streaming.Trigger
|
||||
|
||||
import sys.process._
|
||||
import java.io.File
|
||||
import java.nio.file.{Files, Paths, StandardCopyOption}
|
||||
|
||||
import scala.io.Source
|
||||
import org.apache.commons.io.FilenameUtils
|
||||
|
||||
object Benchmark {
|
||||
|
||||
def main(queryIdx: Int = 1, data_path: String = "~/data/taxi_data_cleaned_18_standard",
|
||||
home_dir: String = "~/git/PipeDream/.image", out_path: String="~/data/fpga_queries",
|
||||
start: Int=0, count: Int = 0, verbose: Boolean = false) = {
|
||||
|
||||
val taxi_files = new File(data_path).listFiles.drop(start)
|
||||
val total = if (count > 0) count else taxi_files.length
|
||||
|
||||
println(s"total files: $total")
|
||||
|
||||
val taxi_idxs = start to (total - 1 + start)
|
||||
val taxi_data_copy = s"${data_path}_copy"
|
||||
|
||||
println(s"total idxs: ${taxi_idxs.size}")
|
||||
|
||||
val out_dir_res = new File(s"${out_path}/q_${queryIdx}/results")
|
||||
val times_dir = new File(s"${out_path}/q_${queryIdx}/monitor")
|
||||
|
||||
if(! out_dir_res.exists()) {
|
||||
out_dir_res.mkdirs()
|
||||
}
|
||||
|
||||
if(! times_dir.exists()) {
|
||||
times_dir.mkdirs()
|
||||
}
|
||||
|
||||
for(file_id <- taxi_idxs zip taxi_files) {
|
||||
file_id match {
|
||||
case (idx, taxi_file) => {
|
||||
// fix the file
|
||||
val lineSize = Source.fromFile(taxi_file).getLines.take(1).toList.head.split(',').size
|
||||
|
||||
val proc =
|
||||
Process(s"${home_dir}/acceltesthw --file_name ${taxi_file} --query_mode=${queryIdx + 1} --accel_type=0 --num_cols=$lineSize",
|
||||
new File(home_dir))
|
||||
|
||||
println(s"Starting. Cols $lineSize, name: $taxi_file")
|
||||
if (verbose) proc.! else proc.!!
|
||||
println(s"Completed")
|
||||
|
||||
val suffix = FilenameUtils.getBaseName(taxi_file.toString)
|
||||
|
||||
Files.move(
|
||||
Paths.get(s"${home_dir}/fpga_time.csv"),
|
||||
Paths.get(s"${times_dir.toString}/fpga_time_${suffix}.csv"),
|
||||
StandardCopyOption.REPLACE_EXISTING
|
||||
)
|
||||
for {
|
||||
files <- Option(new File(home_dir).listFiles())
|
||||
file <- files if file.getName.endsWith(".log")
|
||||
} file.delete()
|
||||
|
||||
// Files.move(
|
||||
// Paths.get(s"${home_dir}/q1out.csv"),
|
||||
// Paths.get(s"${out_dir_res.toString}/q0out_${suffix}.csv"),
|
||||
// StandardCopyOption.REPLACE_EXISTING
|
||||
// )
|
||||
|
||||
Files.move(
|
||||
Paths.get(s"${home_dir}/q${queryIdx + 1}out.csv"),
|
||||
Paths.get(s"${out_dir_res.toString}/q${queryIdx}out_${suffix}.csv"),
|
||||
StandardCopyOption.REPLACE_EXISTING
|
||||
)
|
||||
|
||||
// val file_name = new File(taxi_file.toString).getName
|
||||
// Files.move(
|
||||
// Paths.get(taxi_file.toString),
|
||||
// Paths.get(s"${taxi_data_copy}/${file_name}"),
|
||||
// StandardCopyOption.REPLACE_EXISTING
|
||||
// )
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
|
@ -0,0 +1,66 @@
|
|||
import org.apache.spark.sql.functions.{_ => F}
|
||||
import org.apache.spark.sql.SparkSession
|
||||
import org.apache.spark.sql.functions._
|
||||
import org.apache.spark.sql.types._
|
||||
import org.apache.spark.sql.DataFrame
|
||||
import org.apache.log4j._
|
||||
import java.nio.file.Paths
|
||||
import org.apache.spark.SparkConf
|
||||
|
||||
:load schemas.scala
|
||||
|
||||
object Benchmark {
|
||||
var cols = Array("")
|
||||
|
||||
def getSchema() = Schemas.getTaxiSchema
|
||||
|
||||
def time[R](block: => R): R = {
|
||||
val t0 = System.nanoTime()
|
||||
val result = block // call-by-name
|
||||
val t1 = System.nanoTime()
|
||||
println(f"Elapsed time: ${(t1 - t0) / 1e9}%1.3fs")
|
||||
result
|
||||
}
|
||||
|
||||
def loadTaxiData(fileName: String) = {
|
||||
|
||||
val taxi_orig_schema = getSchema()
|
||||
|
||||
val df = spark
|
||||
.read
|
||||
.schema(taxi_orig_schema)
|
||||
.option("header", "false")
|
||||
.csv(fileName)
|
||||
|
||||
df.createOrReplaceTempView("nyyellowtaxi")
|
||||
cols = df.columns
|
||||
println(s"Created sql table from dataframe")
|
||||
}
|
||||
|
||||
/** Our main function where the action happens */
|
||||
def runFile(file_name: String) = {
|
||||
val schema_fields = cols
|
||||
val total_cols = schema_fields.length
|
||||
|
||||
var i = 0
|
||||
for(i <- 1 to total_cols) {
|
||||
val names = schema_fields.slice(0, i).mkString(", ")
|
||||
val query_txt = spark.sql(s"select $names from nyyellowtaxi")
|
||||
|
||||
println(s"cols ${i}:")
|
||||
time {query_txt.collect()}
|
||||
}
|
||||
}
|
||||
|
||||
def main(rootPath: String = "~/yellow/taxi/taxi_data") = {
|
||||
val files = List("yellow_tripdata_2012-03.csv", "yellow_tripdata_2019-12.csv")
|
||||
|
||||
println(s"spark.driver.memory: ${spark.conf.get("spark.driver.memory")}")
|
||||
var f : String = ""
|
||||
for (f <- files.map(Paths.get(rootPath, _).toString)) {
|
||||
println(s"Running with $f")
|
||||
loadTaxiData(f)
|
||||
runFile(f)
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,20 @@
|
|||
import os
|
||||
import glob
|
||||
from multiprocessing import Pool, cpu_count
|
||||
|
||||
path_root = os.path.expanduser(r"~/data/taxi_data")
|
||||
files = glob.glob(os.path.join(path_root, "*.csv"))
|
||||
|
||||
def fixFile(f):
|
||||
print(f"Reading: {os.path.split(f)[1]}")
|
||||
with open(f, "r") as fd:
|
||||
lines = fd.readlines()
|
||||
|
||||
if len(lines[0].split(',')) < 18:
|
||||
print(f"Will write: {os.path.split(f)[1]}")
|
||||
with open(f, "w") as fd:
|
||||
fd.writelines(lines[1:])
|
||||
|
||||
Pool(cpu_count()).map(fixFile, files)
|
||||
|
||||
|
|
@ -0,0 +1,82 @@
|
|||
13.97956
|
||||
13.093169
|
||||
14.097414
|
||||
15.846028
|
||||
14.911613
|
||||
14.005469
|
||||
13.39619
|
||||
13.455239
|
||||
13.741334
|
||||
15.320745
|
||||
14.549938
|
||||
14.609953
|
||||
14.595407
|
||||
14.979908
|
||||
15.349394
|
||||
14.608228
|
||||
14.411513
|
||||
12.299834
|
||||
15.770246
|
||||
13.928713
|
||||
13.639649
|
||||
13.964589
|
||||
13.515153
|
||||
0.984463
|
||||
15.802848
|
||||
1.002862
|
||||
19.031792
|
||||
1.021823
|
||||
14.604164
|
||||
13.236647
|
||||
15.921361
|
||||
15.627326
|
||||
14.276602
|
||||
14.624606
|
||||
14.722508
|
||||
14.773239
|
||||
15.843256
|
||||
15.191065
|
||||
15.420166
|
||||
15.035725
|
||||
14.145152
|
||||
14.213941
|
||||
15.519243
|
||||
14.478537
|
||||
13.46988
|
||||
14.247559
|
||||
14.295364
|
||||
13.556319
|
||||
15.724502
|
||||
14.577173
|
||||
14.808015
|
||||
14.066502
|
||||
13.398903
|
||||
12.168592
|
||||
13.732333
|
||||
1.024708
|
||||
13.890993
|
||||
13.546112
|
||||
13.30142
|
||||
12.761658
|
||||
14.967752
|
||||
14.205653
|
||||
15.752544
|
||||
13.605788
|
||||
12.791365
|
||||
12.320598
|
||||
13.025826
|
||||
13.799213
|
||||
12.897513
|
||||
12.576882
|
||||
7.382327
|
||||
6.735299
|
||||
7.496559
|
||||
7.114357
|
||||
7.276366
|
||||
6.652565
|
||||
6.770595
|
||||
5.955355
|
||||
0.444866
|
||||
7.558073
|
||||
0.468676
|
||||
7.305288
|
|
|
@ -0,0 +1,82 @@
|
|||
0.947825
|
||||
0.912538
|
||||
0.980564
|
||||
0.979225
|
||||
0.995022
|
||||
0.958112
|
||||
0.949534
|
||||
0.951962
|
||||
1.063844
|
||||
0.967885
|
||||
1.009119
|
||||
1.073668
|
||||
1.00906
|
||||
0.979072
|
||||
0.840295
|
||||
1.068371
|
||||
0.964575
|
||||
0.947867
|
||||
0.948287
|
||||
0.944065
|
||||
1.00033
|
||||
1.081844
|
||||
1.005351
|
||||
1.02761
|
||||
1.012189
|
||||
0.998865
|
||||
0.920604
|
||||
0.99245
|
||||
1.080438
|
||||
1.030567
|
||||
1.028355
|
||||
1.012393
|
||||
1.101289
|
||||
1.056697
|
||||
1.045737
|
||||
1.02883
|
||||
0.978511
|
||||
0.994138
|
||||
0.995688
|
||||
0.927906
|
||||
0.975625
|
||||
0.979342
|
||||
0.955342
|
||||
1.053516
|
||||
0.851215
|
||||
0.962986
|
||||
1.006164
|
||||
0.964013
|
||||
0.955302
|
||||
1.029765
|
||||
0.875359
|
||||
0.984822
|
||||
0.997328
|
||||
0.945503
|
||||
0.904668
|
||||
0.844486
|
||||
0.911973
|
||||
0.949751
|
||||
0.880459
|
||||
0.888663
|
||||
0.51473
|
||||
0.46568
|
||||
0.535507
|
||||
0.512967
|
||||
0.520184
|
||||
0.467202
|
||||
0.42576
|
||||
0.427634
|
||||
0.434916
|
||||
0.491674
|
||||
0.482278
|
||||
0.468268
|
||||
0.918656
|
||||
0.969311
|
||||
1.005039
|
||||
0.978991
|
||||
0.960973
|
||||
1.019734
|
||||
1.015870
|
||||
0.971850
|
||||
0.921574
|
||||
1.075810
|
|
|
@ -0,0 +1,82 @@
|
|||
940475
|
||||
893053
|
||||
966555
|
||||
977252
|
||||
1009676
|
||||
944347
|
||||
903349
|
||||
917968
|
||||
1010628
|
||||
1037121
|
||||
948746
|
||||
984671
|
||||
990188
|
||||
1054078
|
||||
1033425
|
||||
992200
|
||||
971913
|
||||
828716
|
||||
57656680
|
||||
970971
|
||||
927944
|
||||
970230
|
||||
918666
|
||||
995798
|
||||
1073673
|
||||
1031233
|
||||
1040431
|
||||
1019893
|
||||
1022448
|
||||
877899
|
||||
992017
|
||||
1045459
|
||||
967618
|
||||
999996
|
||||
990385
|
||||
997292
|
||||
1084605
|
||||
1042402
|
||||
1032976
|
||||
1026446
|
||||
960292
|
||||
964271
|
||||
982060
|
||||
994965
|
||||
910716
|
||||
970574
|
||||
991765
|
||||
940273
|
||||
1073958
|
||||
1005066
|
||||
1012261
|
||||
970688
|
||||
918614
|
||||
849510
|
||||
958614
|
||||
988098
|
||||
964627
|
||||
946995
|
||||
926526
|
||||
880872
|
||||
1018292
|
||||
978586
|
||||
1017643
|
||||
914257
|
||||
868684
|
||||
840789
|
||||
881766
|
||||
945202
|
||||
873355
|
||||
869790
|
||||
510038
|
||||
463499
|
||||
531702
|
||||
509655
|
||||
497279
|
||||
496703
|
||||
46059508
|
||||
22280323
|
||||
23645268
|
||||
52333855
|
||||
24677348
|
||||
50319654
|
|
|
@ -0,0 +1,82 @@
|
|||
59354080
|
||||
55398376
|
||||
58229390
|
||||
58656620
|
||||
61035949
|
||||
58887203
|
||||
35820233
|
||||
55801987
|
||||
57264781
|
||||
83488417
|
||||
58056335
|
||||
39249339
|
||||
60985527
|
||||
41008864
|
||||
64021800
|
||||
60920472
|
||||
60618628
|
||||
51611609
|
||||
72174898
|
||||
57942850
|
||||
56235630
|
||||
59071567
|
||||
56058763
|
||||
59437963
|
||||
67151400
|
||||
60708282
|
||||
64315256
|
||||
64387971
|
||||
60365695
|
||||
34776418
|
||||
60128128
|
||||
64864423
|
||||
38085689
|
||||
59258581
|
||||
61561927
|
||||
61608793
|
||||
66316890
|
||||
62863393
|
||||
41744434
|
||||
40361009
|
||||
58704546
|
||||
37883628
|
||||
65545393
|
||||
59815354
|
||||
56589525
|
||||
60183791
|
||||
59647102
|
||||
57341955
|
||||
40765435
|
||||
38944168
|
||||
39950603
|
||||
37967729
|
||||
37037386
|
||||
51776137
|
||||
36466504
|
||||
61148758
|
||||
59801669
|
||||
56787193
|
||||
56931899
|
||||
33997980
|
||||
41392988
|
||||
59857394
|
||||
57574703
|
||||
57361239
|
||||
33852335
|
||||
51406059
|
||||
54254661
|
||||
58993313
|
||||
54729486
|
||||
53459283
|
||||
31644699
|
||||
28791572
|
||||
19759028
|
||||
18485513
|
||||
31071754
|
||||
17300642
|
||||
47454626
|
||||
28759663
|
||||
31260343
|
||||
53659689
|
||||
32607724
|
||||
51558698
|
|
|
@ -0,0 +1,82 @@
|
|||
2020-05-07T02:45:07.525Z,14092413,98,1008072.7147349415,14868159.206604594,13.979560000000001,0.947825
|
||||
2020-05-07T02:45:17.757Z,13380122,98,1021916.2373906577,14662536.792988347,13.093169,0.912538
|
||||
2020-05-07T02:45:25.482Z,14387371,98,1020568.0985179268,14672546.616029143,14.097414,0.980564
|
||||
2020-05-07T02:45:29.054Z,14294783,98,902105.1206018316,14598057.647629503,15.846028,0.979225
|
||||
2020-05-07T02:45:35.481Z,14796312,98,992267.7043724243,14870336.535272587,14.911613000000001,0.995022
|
||||
2020-05-07T02:45:42.740Z,14184248,98,1012764.9420379996,14804373.601416117,14.005469,0.9581120000000001
|
||||
2020-05-07T02:45:49.755Z,13626103,98,1017162.5663714833,14350305.518285813,13.396189999999999,0.949534
|
||||
2020-05-07T02:46:04.062Z,13686520,98,1017188.918011787,14377170.517310563,13.455239,0.9519620000000001
|
||||
2020-05-07T02:46:11.376Z,13984887,98,1017724.1161593191,13145618.154541455,13.741334,1.063844
|
||||
2020-05-07T02:46:18.646Z,15604551,98,1018524.2950000147,16122319.283799212,15.320745,0.967885
|
||||
2020-05-07T02:50:57.535Z,14275339,98,981127.1360743942,14146338.538864097,14.549938000000001,1.009119
|
||||
2020-05-07T02:51:05.500Z,14583404,98,998182.8141404699,13582787.230317006,14.609952999999999,1.073668
|
||||
2020-05-07T02:51:12.307Z,14863778,98,1018387.3598043547,14730321.289120566,14.595407000000002,1.00906
|
||||
2020-05-07T02:51:15.936Z,15144990,98,1011020.2278945907,15468719.358739706,14.979907999999998,0.9790719999999999
|
||||
2020-05-07T02:51:23.444Z,15481351,98,1008596.8866262733,18423709.53058152,15.349394,0.8402950000000001
|
||||
2020-05-07T02:51:31.342Z,14825128,98,1014847.7967348264,13876385.637573466,14.608228,1.068371
|
||||
2020-05-07T02:51:38.659Z,14656519,98,1017000.7132491918,15194794.59865744,14.411513000000001,0.964575
|
||||
2020-05-07T02:51:45.848Z,12528177,98,1018564.7220929974,13217230.898427732,12.299833999999999,0.947867
|
||||
2020-05-07T02:51:53.831Z,15698454,98,995447.6296691885,16554538.868507106,15.770245999999998,0.9482870000000001
|
||||
2020-05-07T02:52:00.962Z,14199628,98,1019450.1099993947,15040943.15539714,13.928713,0.9440649999999999
|
||||
2020-05-07T02:52:04.960Z,13912333,98,1019992.0100583235,13907743.444663262,13.639648999999999,1.00033
|
||||
2020-05-07T02:52:08.228Z,14183601,98,1015683.3831629416,13110578.789548216,13.964589000000002,1.081844
|
||||
2020-05-07T02:52:15.634Z,13724125,98,1015462.0521129136,13651078.081187563,13.515153,1.005351
|
||||
2020-05-07T02:52:22.751Z,14612198,98,14842810.75063258,14219594.982532281,0.9844629999999999,1.0276100000000001
|
||||
2020-05-07T02:52:30.684Z,16066350,98,1016674.3361702902,15872875.520283267,15.802848000000001,1.012189
|
||||
2020-05-07T02:52:38.249Z,14718972,98,14676966.521814564,14735697.016113289,1.0028620000000001,0.998865
|
||||
2020-05-07T02:52:45.027Z,15554868,98,817309.6889667561,16896372.381610334,19.031792000000003,0.9206040000000001
|
||||
2020-05-07T02:52:48.857Z,15097860,98,14775416.094568238,15212716.005844122,1.021823,0.99245
|
||||
2020-05-07T02:52:55.937Z,14742561,98,1009476.5438131207,13644985.644710757,14.604164,1.080438
|
||||
2020-05-07T02:53:02.841Z,13262441,98,1001948.680810178,12869072.073916592,13.236647,1.030567
|
||||
2020-05-07T02:53:09.825Z,14626747,98,918686.976571915,14223441.321333587,15.921361,1.028355
|
||||
2020-05-07T02:53:17.561Z,15707756,98,1005146.7538336373,15515472.746255653,15.627326,1.012393
|
||||
2020-05-07T02:53:21.195Z,14525862,98,1017459.3366124517,13189872.95796108,14.276601999999999,1.101289
|
||||
2020-05-07T02:53:27.818Z,14925983,98,1020607.5295293425,14125130.477326991,14.624606,1.056697
|
||||
2020-05-07T02:53:35.324Z,14969131,98,1016751.4257760975,14314431.831330437,14.722508,1.0457370000000001
|
||||
2020-05-07T02:53:41.919Z,14983521,98,1014233.9807810595,14563650.943304531,14.773239000000002,1.02883
|
||||
2020-05-07T02:53:49.568Z,16146923,98,1019166.9565902363,16501524.25470945,15.843256,0.978511
|
||||
2020-05-07T02:53:56.946Z,15477914,98,1018882.7445607006,15569180.536303813,15.191065,0.994138
|
||||
2020-05-07T02:54:04.143Z,15567525,98,1009556.2525072687,15634942.873671273,15.420166,0.9956879999999999
|
||||
2020-05-07T02:54:12.022Z,15096468,98,1004039.9116105144,16269393.667030929,15.035725,0.9279059999999999
|
||||
2020-05-07T02:54:19.687Z,14379307,98,1016553.7280900199,14738559.38500961,14.145152,0.975625
|
||||
2020-05-07T02:54:27.288Z,14381752,98,1011806.0853073753,14685117.150086487,14.213941,0.979342
|
||||
2020-05-07T02:46:25.939Z,14546853,98,937342.9490085309,15226853.838730004,15.519243,0.955342
|
||||
2020-05-07T02:46:33.477Z,14522315,98,1003023.6480384724,13784617.414448382,14.478537,1.053516
|
||||
2020-05-07T02:46:40.998Z,13776030,98,1022728.4875589093,16183960.574003045,13.469879999999998,0.8512149999999999
|
||||
2020-05-07T02:46:47.558Z,14696583,98,1031515.8547509789,15261471.090960823,14.247558999999999,0.962986
|
||||
2020-05-07T02:46:55.724Z,14776615,98,1033664.830080577,14686089.94159998,14.295364000000001,1.0061639999999998
|
||||
2020-05-07T02:47:02.490Z,13990176,98,1032004.0418051537,14512434.99828322,13.556319,0.9640129999999999
|
||||
2020-05-07T02:47:10.375Z,15749228,98,1001572.4504343604,16486124.806605658,15.724502,0.9553020000000001
|
||||
2020-05-07T02:47:16.786Z,15100468,98,1035898.2499555983,14663994.212271731,14.577173000000002,1.029765
|
||||
2020-05-07T02:47:23.712Z,15285049,98,1032214.5810900381,17461463.239653673,14.808015,0.875359
|
||||
2020-05-07T02:47:27.799Z,14385456,98,1022674.7204102342,14607163.528028416,14.066502,0.984822
|
||||
2020-05-07T02:47:34.950Z,13823840,98,1031714.3127314228,13860876.26137038,13.398903,0.9973280000000001
|
||||
2020-05-07T02:47:42.488Z,12597108,98,1035214.9205101132,13323181.417721571,12.168592,0.9455030000000001
|
||||
2020-05-07T02:47:46.145Z,14107693,98,1027334.0298403775,15594331.843283944,13.732332999999999,0.904668
|
||||
2020-05-07T02:47:53.560Z,15004555,98,14642761.645268701,17767677.61691727,1.024708,0.8444860000000001
|
||||
2020-05-07T02:48:01.140Z,14388450,98,1035811.4787042222,15777276.300943121,13.890993,0.9119729999999999
|
||||
2020-05-07T02:48:07.768Z,13971118,98,1031374.7590452523,14710295.645911401,13.546111999999999,0.949751
|
||||
2020-05-07T02:48:15.195Z,13782492,98,1036166.9656322406,15653757.869474899,13.301420000000002,0.880459
|
||||
2020-05-07T02:48:22.418Z,13063791,98,1023675.0585229599,14700500.639725072,12.761658,0.8886629999999999
|
||||
2020-05-07T02:48:26.331Z,15428127,98,1030757.7918180366,29973242.282361627,14.967751999999999,0.51473
|
||||
2020-05-07T02:48:33.426Z,14618759,98,1029080.394966708,31392284.401305616,14.205653,0.46568000000000004
|
||||
2020-05-07T02:48:40.749Z,14774041,98,937882.8587941097,27588884.925873987,15.752544,0.535507
|
||||
2020-05-07T02:48:48.612Z,13813029,98,1015231.8263374381,26927714.648310706,13.605788,0.5129670000000001
|
||||
2020-05-07T02:48:56.043Z,13106365,98,1024625.9879223211,25195632.699198745,12.791364999999999,0.520184
|
||||
2020-05-07T02:49:03.307Z,12688877,98,1029891.3250801625,27159295.122880463,12.320598,0.46720200000000006
|
||||
2020-05-07T02:49:10.732Z,13374016,98,1026730.7424496535,31412100.714017283,13.025826,0.42576
|
||||
2020-05-07T02:49:17.942Z,14232487,98,1031398.4572888323,33281935.01919866,13.799213,0.427634
|
||||
2020-05-07T02:49:26.123Z,13218216,98,1024865.4915098748,30392572.35880032,12.897513,0.43491599999999997
|
||||
2020-05-07T02:49:33.034Z,13014161,98,1034768.4744120203,26469085.20686471,12.576882000000001,0.491674
|
||||
2020-05-07T02:49:40.349Z,7667792,98,1038668.7016167125,15899112.130348057,7.382327,0.482278
|
||||
2020-05-07T02:49:44.253Z,7019375,98,1042177.1921335637,14990080.466741268,6.735299,0.468268
|
||||
2020-05-07T02:49:48.217Z,7832545,98,1044818.6961511276,8526091.376968093,7.4965589999999995,0.9186559999999999
|
||||
2020-05-07T02:49:54.664Z,7433139,98,1044808.2658770145,7668476.887191004,7.114357000000001,0.969311
|
||||
2020-05-07T02:50:02.789Z,7565261,98,1039703.1979974619,7527330.78019858,7.276366,1.005039
|
||||
2020-05-07T02:50:06.505Z,6941024,98,1043360.5684423979,7089977.333805929,6.652564999999999,0.978991
|
||||
2020-05-07T02:50:13.912Z,6310417,98,932032.8567873281,6566695.422243914,6.770595,0.960973
|
||||
2020-05-07T02:50:21.053Z,6073357,98,1019814.4359152393,5955824.75429867,5.955355,1.019734
|
||||
2020-05-07T02:50:28.426Z,6567787,98,14763517.55360041,6465184.521641549,0.444866,1.01587
|
||||
2020-05-07T02:50:35.988Z,7213889,98,954461.408351044,7422842.00236662,7.558072999999999,0.97185
|
||||
2020-05-07T02:50:43.080Z,6878111,98,14675620.257918049,7463438.6386768725,0.46867600000000004,0.921574
|
||||
2020-05-07T02:50:49.974Z,6896315,98,944016.8546400907,6410346.6225448735,7.305288000000001,1.07581
|
|
|
@ -0,0 +1,82 @@
|
|||
2020-05-07T02:45:07.525Z,14092413,98,1008072.7147349415,14984356.840958027,13.979560000000001,0.940475
|
||||
2020-05-07T02:45:17.757Z,13380122,98,1021916.2373906577,14982450.089748314,13.093169,0.893053
|
||||
2020-05-07T02:45:25.482Z,14387371,98,1020568.0985179268,14885206.739399206,14.097414,0.966555
|
||||
2020-05-07T02:45:29.054Z,14294783,98,902105.1206018316,14627530.053660672,15.846028,0.977252
|
||||
2020-05-07T02:45:35.481Z,14796312,98,992267.7043724243,14654514.913695086,14.911613000000001,1.009676
|
||||
2020-05-07T02:45:42.740Z,14184248,98,1012764.9420379996,15020165.257050639,14.005469,0.944347
|
||||
2020-05-07T02:45:49.755Z,13626103,98,1017162.5663714833,15083985.259296242,13.396189999999999,0.903349
|
||||
2020-05-07T02:46:04.062Z,13686520,98,1017188.918011787,14909582.904850714,13.455239,0.917968
|
||||
2020-05-07T02:46:11.376Z,13984887,98,1017724.1161593191,13837818.66324701,13.741334,1.010628
|
||||
2020-05-07T02:46:18.646Z,15604551,98,1018524.2950000147,15046027.41628026,15.320745,1.037121
|
||||
2020-05-07T02:50:57.535Z,14275339,98,981127.1360743942,15046534.056533571,14.549938000000001,0.948746
|
||||
2020-05-07T02:51:05.500Z,14583404,98,998182.8141404699,14810433.129441205,14.609952999999999,0.984671
|
||||
2020-05-07T02:51:12.307Z,14863778,98,1018387.3598043547,15011066.58533531,14.595407000000002,0.990188
|
||||
2020-05-07T02:51:15.936Z,15144990,98,1011020.2278945907,14367997.434724944,14.979907999999998,1.054078
|
||||
2020-05-07T02:51:23.444Z,15481351,98,1008596.8866262733,14980623.654353242,15.349394,1.033425
|
||||
2020-05-07T02:51:31.342Z,14825128,98,1014847.7967348264,14941673.04978835,14.608228,0.9922
|
||||
2020-05-07T02:51:38.659Z,14656519,98,1017000.7132491918,15080073.01065013,14.411513000000001,0.971913
|
||||
2020-05-07T02:51:45.848Z,12528177,98,1018564.7220929974,15117575.864349186,12.299833999999999,0.828716
|
||||
2020-05-07T02:51:53.831Z,15698454,98,995447.6296691885,272274.67832001427,15.770245999999998,57.65668
|
||||
2020-05-07T02:52:00.962Z,14199628,98,1019450.1099993947,14624152.523607811,13.928713,0.970971
|
||||
2020-05-07T02:52:04.960Z,13912333,98,1019992.0100583235,14992642.875001078,13.639648999999999,0.927944
|
||||
2020-05-07T02:52:08.228Z,14183601,98,1015683.3831629416,14618802.7581089,13.964589000000002,0.97023
|
||||
2020-05-07T02:52:15.634Z,13724125,98,1015462.0521129136,14939188.997960085,13.515153,0.918666
|
||||
2020-05-07T02:52:22.751Z,14612198,98,14842810.75063258,14673857.549422674,0.9844629999999999,0.995798
|
||||
2020-05-07T02:52:30.684Z,16066350,98,1016674.3361702902,14963913.593803698,15.802848000000001,1.073673
|
||||
2020-05-07T02:52:38.249Z,14718972,98,14676966.521814564,14273177.836628579,1.0028620000000001,1.031233
|
||||
2020-05-07T02:52:45.027Z,15554868,98,817309.6889667561,14950408.052047659,19.031792000000003,1.040431
|
||||
2020-05-07T02:52:48.857Z,15097860,98,14775416.094568238,14803376.432625776,1.021823,1.019893
|
||||
2020-05-07T02:52:55.937Z,14742561,98,1009476.5438131207,14418885.850429557,14.604164,1.022448
|
||||
2020-05-07T02:53:02.841Z,13262441,98,1001948.680810178,15107023.70090409,13.236647,0.877899
|
||||
2020-05-07T02:53:09.825Z,14626747,98,918686.976571915,14744451.959996652,15.921361,0.992017
|
||||
2020-05-07T02:53:17.561Z,15707756,98,1005146.7538336373,15024746.068473274,15.627326,1.045459
|
||||
2020-05-07T02:53:21.195Z,14525862,98,1017459.3366124517,15011979.934230244,14.276601999999999,0.967618
|
||||
2020-05-07T02:53:27.818Z,14925983,98,1020607.5295293425,14926042.704170818,14.624606,0.999996
|
||||
2020-05-07T02:53:35.324Z,14969131,98,1016751.4257760975,15114456.499240195,14.722508,0.990385
|
||||
2020-05-07T02:53:41.919Z,14983521,98,1014233.9807810595,15024206.551341033,14.773239000000002,0.997292
|
||||
2020-05-07T02:53:49.568Z,16146923,98,1019166.9565902363,14887376.510342475,15.843256,1.084605
|
||||
2020-05-07T02:53:56.946Z,15477914,98,1018882.7445607006,14848315.716969077,15.191065,1.042402
|
||||
2020-05-07T02:54:04.143Z,15567525,98,1009556.2525072687,15070558.270472888,15.420166,1.032976
|
||||
2020-05-07T02:54:12.022Z,15096468,98,1004039.9116105144,14707513.108336922,15.035725,1.026446
|
||||
2020-05-07T02:54:19.687Z,14379307,98,1016553.7280900199,14973890.233387344,14.145152,0.960292
|
||||
2020-05-07T02:54:27.288Z,14381752,98,1011806.0853073753,14914637.067795256,14.213941,0.964271
|
||||
2020-05-07T02:46:25.939Z,14546853,98,937342.9490085309,14812590.880394273,15.519243,0.98206
|
||||
2020-05-07T02:46:33.477Z,14522315,98,1003023.6480384724,14595804.877558507,14.478537,0.994965
|
||||
2020-05-07T02:46:40.998Z,13776030,98,1022728.4875589093,15126592.702884324,13.469879999999998,0.910716
|
||||
2020-05-07T02:46:47.558Z,14696583,98,1031515.8547509789,15142156.084955912,14.247558999999999,0.970574
|
||||
2020-05-07T02:46:55.724Z,14776615,98,1033664.830080577,14899310.824640919,14.295364000000001,0.991765
|
||||
2020-05-07T02:47:02.490Z,13990176,98,1032004.0418051537,14878844.761042804,13.556319,0.940273
|
||||
2020-05-07T02:47:10.375Z,15749228,98,1001572.4504343604,14664659.139370441,15.724502,1.073958
|
||||
2020-05-07T02:47:16.786Z,15100468,98,1035898.2499555983,15024354.619497625,14.577173000000002,1.005066
|
||||
2020-05-07T02:47:23.712Z,15285049,98,1032214.5810900381,15099909.015560215,14.808015,1.012261
|
||||
2020-05-07T02:47:27.799Z,14385456,98,1022674.7204102342,14819855.607569065,14.066502,0.970688
|
||||
2020-05-07T02:47:34.950Z,13823840,98,1031714.3127314228,15048584.062511565,13.398903,0.918614
|
||||
2020-05-07T02:47:42.488Z,12597108,98,1035214.9205101132,14828675.354027616,12.168592,0.84951
|
||||
2020-05-07T02:47:46.145Z,14107693,98,1027334.0298403775,14716760.86516575,13.732332999999999,0.958614
|
||||
2020-05-07T02:47:53.560Z,15004555,98,14642761.645268701,15185290.325453548,1.024708,0.988098
|
||||
2020-05-07T02:48:01.140Z,14388450,98,1035811.4787042222,14916076.369415328,13.890993,0.964627
|
||||
2020-05-07T02:48:07.768Z,13971118,98,1031374.7590452523,14753106.404996859,13.546111999999999,0.946995
|
||||
2020-05-07T02:48:15.195Z,13782492,98,1036166.9656322406,14875450.877795119,13.301420000000002,0.926526
|
||||
2020-05-07T02:48:22.418Z,13063791,98,1023675.0585229599,14830521.347028853,12.761658,0.880872
|
||||
2020-05-07T02:48:26.331Z,15428127,98,1030757.7918180366,15150985.179103833,14.967751999999999,1.018292
|
||||
2020-05-07T02:48:33.426Z,14618759,98,1029080.394966708,14938655.36600769,14.205653,0.978586
|
||||
2020-05-07T02:48:40.749Z,14774041,98,937882.8587941097,14517901.660995062,15.752544,1.017643
|
||||
2020-05-07T02:48:48.612Z,13813029,98,1015231.8263374381,15108474.969292004,13.605788,0.914257
|
||||
2020-05-07T02:48:56.043Z,13106365,98,1024625.9879223211,15087609.53350125,12.791364999999999,0.868684
|
||||
2020-05-07T02:49:03.307Z,12688877,98,1029891.3250801625,15091630.599353703,12.320598,0.840789
|
||||
2020-05-07T02:49:10.732Z,13374016,98,1026730.7424496535,15167307.426233264,13.025826,0.881766
|
||||
2020-05-07T02:49:17.942Z,14232487,98,1031398.4572888323,15057614.139623065,13.799213,0.945202
|
||||
2020-05-07T02:49:26.123Z,13218216,98,1024865.4915098748,15134986.345758598,12.897513,0.873355
|
||||
2020-05-07T02:49:33.034Z,13014161,98,1034768.4744120203,14962417.36511112,12.576882000000001,0.86979
|
||||
2020-05-07T02:49:40.349Z,7667792,98,1038668.7016167125,15033766.11154463,7.382327,0.510038
|
||||
2020-05-07T02:49:44.253Z,7019375,98,1042177.1921335637,15144315.30596614,6.735299,0.463499
|
||||
2020-05-07T02:49:48.217Z,7832545,98,1044818.6961511276,14731080.56768641,7.4965589999999995,0.531702
|
||||
2020-05-07T02:49:54.664Z,7433139,98,1044808.2658770145,14584648.438649675,7.114357000000001,0.509655
|
||||
2020-05-07T02:50:02.789Z,7565261,98,1039703.1979974619,15213312.848521654,7.276366,0.497279
|
||||
2020-05-07T02:50:06.505Z,6941024,98,1043360.5684423979,13974193.834142335,6.652564999999999,0.496703
|
||||
2020-05-07T02:50:13.912Z,6310417,98,932032.8567873281,137005.74048685018,6.770595,46.059508
|
||||
2020-05-07T02:50:21.053Z,6073357,98,1019814.4359152393,272588.3731577859,5.955355,22.280323
|
||||
2020-05-07T02:50:28.426Z,6567787,98,14763517.55360041,277763.27170408895,0.444866,23.645268
|
||||
2020-05-07T02:50:35.988Z,7213889,98,954461.408351044,137843.63869239137,7.558072999999999,52.333855
|
||||
2020-05-07T02:50:43.080Z,6878111,98,14675620.257918049,278721.6438330407,0.46867600000000004,24.677348
|
||||
2020-05-07T02:50:49.974Z,6896315,98,944016.8546400907,137050.12757043203,7.305288000000001,50.319654
|
|
|
@ -0,0 +1,82 @@
|
|||
2020-05-07T02:45:07.525Z,14092413,98,1008072.7147349415,237429.55833870222,13.979560000000001,59.35408
|
||||
2020-05-07T02:45:17.757Z,13380122,98,1021916.2373906577,241525.52775193265,13.093169,55.398376
|
||||
2020-05-07T02:45:25.482Z,14387371,98,1020568.0985179268,247080.9156681875,14.097414,58.22939
|
||||
2020-05-07T02:45:29.054Z,14294783,98,902105.1206018316,243702.80796950115,15.846028,58.65662
|
||||
2020-05-07T02:45:35.481Z,14796312,98,992267.7043724243,242419.62716103587,14.911613000000001,61.035949
|
||||
2020-05-07T02:45:42.740Z,14184248,98,1012764.9420379996,240871.4844208172,14.005469,58.887203
|
||||
2020-05-07T02:45:49.755Z,13626103,98,1017162.5663714833,380402.4111177613,13.396189999999999,35.820233
|
||||
2020-05-07T02:46:04.062Z,13686520,98,1017188.918011787,245269.40232432942,13.455239,55.801987
|
||||
2020-05-07T02:46:11.376Z,13984887,98,1017724.1161593191,244214.45006486622,13.741334,57.264781
|
||||
2020-05-07T02:46:18.646Z,15604551,98,1018524.2950000147,186906.77773900062,15.320745,83.488417
|
||||
2020-05-07T02:50:57.535Z,14275339,98,981127.1360743942,245887.7054502321,14.549938000000001,58.056335
|
||||
2020-05-07T02:51:05.500Z,14583404,98,998182.8141404699,371557.951587414,14.609952999999999,39.249339
|
||||
2020-05-07T02:51:12.307Z,14863778,98,1018387.3598043547,243726.31886906546,14.595407000000002,60.985527
|
||||
2020-05-07T02:51:15.936Z,15144990,98,1011020.2278945907,369310.1569455813,14.979907999999998,41.008864
|
||||
2020-05-07T02:51:23.444Z,15481351,98,1008596.8866262733,241813.74156927798,15.349394,64.0218
|
||||
2020-05-07T02:51:31.342Z,14825128,98,1014847.7967348264,243352.15262284904,14.608228,60.920472
|
||||
2020-05-07T02:51:38.659Z,14656519,98,1017000.7132491918,241782.42701236985,14.411513000000001,60.618628
|
||||
2020-05-07T02:51:45.848Z,12528177,98,1018564.7220929974,242739.51622008142,12.299833999999999,51.611609
|
||||
2020-05-07T02:51:53.831Z,15698454,98,995447.6296691885,217505.73170190002,15.770245999999998,72.174898
|
||||
2020-05-07T02:52:00.962Z,14199628,98,1019450.1099993947,245062.64362212076,13.928713,57.94285
|
||||
2020-05-07T02:52:04.960Z,13912333,98,1019992.0100583235,247393.56525391465,13.639648999999999,56.23563
|
||||
2020-05-07T02:52:08.228Z,14183601,98,1015683.3831629416,240108.7650171867,13.964589000000002,59.071567
|
||||
2020-05-07T02:52:15.634Z,13724125,98,1015462.0521129136,244816.76486511127,13.515153,56.058763
|
||||
2020-05-07T02:52:22.751Z,14612198,98,14842810.75063258,245839.4814102226,0.9844629999999999,59.437963
|
||||
2020-05-07T02:52:30.684Z,16066350,98,1016674.3361702902,239255.62236974956,15.802848000000001,67.1514
|
||||
2020-05-07T02:52:38.249Z,14718972,98,14676966.521814564,242454.10206139585,1.0028620000000001,60.708282
|
||||
2020-05-07T02:52:45.027Z,15554868,98,817309.6889667561,241853.47252602087,19.031792000000003,64.315256
|
||||
2020-05-07T02:52:48.857Z,15097860,98,14775416.094568238,234482.61787904456,1.021823,64.387971
|
||||
2020-05-07T02:52:55.937Z,14742561,98,1009476.5438131207,244220.8443056938,14.604164,60.365695
|
||||
2020-05-07T02:53:02.841Z,13262441,98,1001948.680810178,381363.0546998831,13.236647,34.776418
|
||||
2020-05-07T02:53:09.825Z,14626747,98,918686.976571915,243259.6438059738,15.921361,60.128128
|
||||
2020-05-07T02:53:17.561Z,15707756,98,1005146.7538336373,242162.8879671064,15.627326,64.864423
|
||||
2020-05-07T02:53:21.195Z,14525862,98,1017459.3366124517,381399.48052403616,14.276601999999999,38.085689
|
||||
2020-05-07T02:53:27.818Z,14925983,98,1020607.5295293425,251878.84603581717,14.624606,59.258581
|
||||
2020-05-07T02:53:35.324Z,14969131,98,1016751.4257760975,243155.6601533932,14.722508,61.561927
|
||||
2020-05-07T02:53:41.919Z,14983521,98,1014233.9807810595,243204.26144365466,14.773239000000002,61.608793
|
||||
2020-05-07T02:53:49.568Z,16146923,98,1019166.9565902363,243481.30619514876,15.843256,66.31689
|
||||
2020-05-07T02:53:56.946Z,15477914,98,1018882.7445607006,246215.0587385571,15.191065,62.863393
|
||||
2020-05-07T02:54:04.143Z,15567525,98,1009556.2525072687,372924.56762020057,15.420166,41.744434
|
||||
2020-05-07T02:54:12.022Z,15096468,98,1004039.9116105144,374035.9414701451,15.035725,40.361009
|
||||
2020-05-07T02:54:19.687Z,14379307,98,1016553.7280900199,244943.67097226167,14.145152,58.704546
|
||||
2020-05-07T02:54:27.288Z,14381752,98,1011806.0853073753,379629.74401501357,14.213941,37.883628
|
||||
2020-05-07T02:46:25.939Z,14546853,98,937342.9490085309,221935.55235834804,15.519243,65.545393
|
||||
2020-05-07T02:46:33.477Z,14522315,98,1003023.6480384724,242785.74026327755,14.478537,59.815354
|
||||
2020-05-07T02:46:40.998Z,13776030,98,1022728.4875589093,243437.80938256683,13.469879999999998,56.589525
|
||||
2020-05-07T02:46:47.558Z,14696583,98,1031515.8547509789,244195.035836144,14.247558999999999,60.183791
|
||||
2020-05-07T02:46:55.724Z,14776615,98,1033664.830080577,247733.99720241228,14.295364000000001,59.647102
|
||||
2020-05-07T02:47:02.490Z,13990176,98,1032004.0418051537,243978.00877211112,13.556319,57.341955
|
||||
2020-05-07T02:47:10.375Z,15749228,98,1001572.4504343604,386337.78837390064,15.724502,40.765435
|
||||
2020-05-07T02:47:16.786Z,15100468,98,1035898.2499555983,387746.58120825695,14.577173000000002,38.944168
|
||||
2020-05-07T02:47:23.712Z,15285049,98,1032214.5810900381,382598.70570664475,14.808015,39.950603
|
||||
2020-05-07T02:47:27.799Z,14385456,98,1022674.7204102342,378886.3958652887,14.066502,37.967729
|
||||
2020-05-07T02:47:34.950Z,13823840,98,1031714.3127314228,373240.1633311811,13.398903,37.037386
|
||||
2020-05-07T02:47:42.488Z,12597108,98,1035214.9205101132,243299.49528679592,12.168592,51.776137
|
||||
2020-05-07T02:47:46.145Z,14107693,98,1027334.0298403775,386867.164453165,13.732332999999999,36.466504
|
||||
2020-05-07T02:47:53.560Z,15004555,98,14642761.645268701,245377.91920483488,1.024708,61.148758
|
||||
2020-05-07T02:48:01.140Z,14388450,98,1035811.4787042222,240602.81662038565,13.890993,59.801669
|
||||
2020-05-07T02:48:07.768Z,13971118,98,1031374.7590452523,246025.86009137658,13.546111999999999,56.787193
|
||||
2020-05-07T02:48:15.195Z,13782492,98,1036166.9656322406,242087.3401746181,13.301420000000002,56.931899
|
||||
2020-05-07T02:48:22.418Z,13063791,98,1023675.0585229599,384251.97614681814,12.761658,33.99798
|
||||
2020-05-07T02:48:26.331Z,15428127,98,1030757.7918180366,372723.2013306215,14.967751999999999,41.392988
|
||||
2020-05-07T02:48:33.426Z,14618759,98,1029080.394966708,244226.4526250508,14.205653,59.857394
|
||||
2020-05-07T02:48:40.749Z,14774041,98,937882.8587941097,256606.4648218854,15.752544,57.574703
|
||||
2020-05-07T02:48:48.612Z,13813029,98,1015231.8263374381,240807.71686260126,13.605788,57.361239
|
||||
2020-05-07T02:48:56.043Z,13106365,98,1024625.9879223211,387162.80575623515,12.791364999999999,33.852335
|
||||
2020-05-07T02:49:03.307Z,12688877,98,1029891.3250801625,246836.21438476737,12.320598,51.406059
|
||||
2020-05-07T02:49:10.732Z,13374016,98,1026730.7424496535,246504.46161667106,13.025826,54.254661
|
||||
2020-05-07T02:49:17.942Z,14232487,98,1031398.4572888323,241255.9369228848,13.799213,58.993313
|
||||
2020-05-07T02:49:26.123Z,13218216,98,1024865.4915098748,241519.09630578294,12.897513,54.729486
|
||||
2020-05-07T02:49:33.034Z,13014161,98,1034768.4744120203,243440.62003225894,12.576882000000001,53.459283
|
||||
2020-05-07T02:49:40.349Z,7667792,98,1038668.7016167125,242308.89350535456,7.382327,31.644699
|
||||
2020-05-07T02:49:44.253Z,7019375,98,1042177.1921335637,243799.64386800418,6.735299,28.791572
|
||||
2020-05-07T02:49:48.217Z,7832545,98,1044818.6961511276,396403.3554687002,7.4965589999999995,19.759028
|
||||
2020-05-07T02:49:54.664Z,7433139,98,1044808.2658770145,402106.17903868825,7.114357000000001,18.485513
|
||||
2020-05-07T02:50:02.789Z,7565261,98,1039703.1979974619,243477.1142948673,7.276366,31.071754
|
||||
2020-05-07T02:50:06.505Z,6941024,98,1043360.5684423979,401200.37163938774,6.652564999999999,17.300642
|
||||
2020-05-07T02:50:13.912Z,6310417,98,932032.8567873281,132977.9103095239,6.770595,47.454626
|
||||
2020-05-07T02:50:21.053Z,6073357,98,1019814.4359152393,211176.2227533751,5.955355,28.759663
|
||||
2020-05-07T02:50:28.426Z,6567787,98,14763517.55360041,210099.6460595458,0.444866,31.260343
|
||||
2020-05-07T02:50:35.988Z,7213889,98,954461.408351044,134437.77134079178,7.558072999999999,53.659689
|
||||
2020-05-07T02:50:43.080Z,6878111,98,14675620.257918049,210935.02263451446,0.46867600000000004,32.607724
|
||||
2020-05-07T02:50:49.974Z,6896315,98,944016.8546400907,133756.57779410953,7.305288000000001,51.558698
|
|
|
@ -0,0 +1,82 @@
|
|||
2020-05-07T02:45:07.525Z,15097860,98,0.0,1477430.276935121,0.0,10.219
|
||||
2020-05-07T02:45:17.757Z,14612198,98,1428088.1548084442,1891791.5587778352,10.232,7.724
|
||||
2020-05-07T02:45:25.482Z,6567787,98,850198.9644012946,1839201.0641276953,7.724999999999999,3.571
|
||||
2020-05-07T02:45:29.054Z,12528177,98,3507328.387458007,1949607.3762838468,3.572,6.426
|
||||
2020-05-07T02:45:35.481Z,13724125,98,2135385.8721020697,1890896.2524111324,6.427,7.258
|
||||
2020-05-07T02:45:42.740Z,13686520,98,1885455.2968728475,1951314.5138294839,7.259,7.014
|
||||
2020-05-07T02:45:49.755Z,14388450,98,2051097.647897363,1005763.3160911506,7.015,14.306
|
||||
2020-05-07T02:46:04.062Z,14379307,98,1005053.9596001956,1966266.5116915083,14.307,7.313
|
||||
2020-05-07T02:46:11.376Z,14184248,98,1939328.411266065,1951065.7496561212,7.314,7.27
|
||||
2020-05-07T02:46:18.646Z,14387371,98,1979005.6396148556,1973034.9698299507,7.27,7.292
|
||||
2020-05-07T02:50:57.535Z,15707756,98,2077470.7049332098,1972345.0527373177,7.561,7.964
|
||||
2020-05-07T02:51:05.500Z,13380122,98,1679864.6578782173,1965641.5454679006,7.965,6.807
|
||||
2020-05-07T02:51:12.307Z,6878111,98,1010446.745996768,1895318.5450537337,6.807,3.629
|
||||
2020-05-07T02:51:15.936Z,14742561,98,4062430.6971617527,1963841.8809111498,3.629,7.507
|
||||
2020-05-07T02:51:23.444Z,15554868,98,2071772.5093233883,1969718.6273268329,7.508,7.897
|
||||
2020-05-07T02:51:31.342Z,14294783,98,1809924.4112433528,1953639.8797321306,7.898,7.317
|
||||
2020-05-07T02:51:38.659Z,14183601,98,1938444.8544485443,1972958.8259841425,7.317,7.189
|
||||
2020-05-07T02:51:45.848Z,15567525,98,2165464.598692447,1950084.5546786922,7.188999999999999,7.983
|
||||
2020-05-07T02:51:53.831Z,13971118,98,1750108.7310534888,1959208.8066189873,7.983,7.131
|
||||
2020-05-07T02:52:00.962Z,7832545,98,1098379.6101528537,1959605.9544658496,7.131,3.9969999999999994
|
||||
2020-05-07T02:52:04.960Z,6073357,98,1519098.7993996998,1859001.2243648607,3.998,3.267
|
||||
2020-05-07T02:52:08.228Z,14525862,98,4444878.212974297,1961628.8993923024,3.2679999999999993,7.405
|
||||
2020-05-07T02:52:15.634Z,14107693,98,1904900.4860923577,1982531.3378302418,7.406,7.116
|
||||
2020-05-07T02:52:22.751Z,15604551,98,2192574.2588169174,1967538.8979952086,7.116999999999999,7.931
|
||||
2020-05-07T02:52:30.684Z,14863778,98,1873664.1875709065,1964808.7243886318,7.933,7.565
|
||||
2020-05-07T02:52:38.249Z,13262441,98,1753131.658955717,1956689.436411921,7.565,6.778
|
||||
2020-05-07T02:52:45.027Z,7433139,98,1096656.68338743,1940767.362924282,6.778,3.83
|
||||
2020-05-07T02:52:48.857Z,13990176,98,3652787.467362924,1976292.6967085747,3.83,7.079
|
||||
2020-05-07T02:52:55.937Z,13626103,98,1924590.8192090394,1973653.3893395134,7.08,6.904
|
||||
2020-05-07T02:53:02.841Z,13823840,98,2002294.3221320973,1979358.5337915234,6.904,6.984
|
||||
2020-05-07T02:53:09.825Z,15096468,98,2161579.0378006874,1951708.8558500323,6.984,7.735
|
||||
2020-05-07T02:53:17.561Z,6941024,98,897236.8148914168,1910548.8576933665,7.736,3.633
|
||||
2020-05-07T02:53:21.195Z,13106365,98,3606594.6615299946,1979215.493808517,3.634,6.622
|
||||
2020-05-07T02:53:27.818Z,15100468,98,2280004.2276913784,2011786.3042899014,6.623,7.506
|
||||
2020-05-07T02:53:35.324Z,13063791,98,1740446.4428457234,1980862.926459439,7.506,6.595
|
||||
2020-05-07T02:53:41.919Z,15285049,98,2317672.327520849,1998306.8374950974,6.595,7.649
|
||||
2020-05-07T02:53:49.568Z,14381752,98,1880213.3612236893,1949539.3791514167,7.649,7.377
|
||||
2020-05-07T02:53:56.946Z,14385456,98,1949777.1753862835,1999090.6058921623,7.378,7.196
|
||||
2020-05-07T02:54:04.143Z,15749228,98,2188304.5713491733,1998886.6607437492,7.197,7.879
|
||||
2020-05-07T02:54:12.022Z,15428127,98,1958132.631044549,2013064.5876826723,7.879,7.664
|
||||
2020-05-07T02:54:19.687Z,15144990,98,1975863.01369863,1992761.8421052631,7.665,7.6
|
||||
2020-05-07T02:54:27.288Z,14583404,98,1918616.497829233,1969665.586169638,7.601,7.404
|
||||
2020-05-07T02:46:25.939Z,15004555,98,2057391.33415604,1990786.1217991244,7.293,7.537
|
||||
2020-05-07T02:46:33.477Z,14925983,98,1980098.56725922,1984838.1648936172,7.538,7.52
|
||||
2020-05-07T02:46:40.998Z,13014161,98,1730376.4127110757,1984168.4708034762,7.521,6.559
|
||||
2020-05-07T02:46:47.558Z,16146923,98,2461421.18902439,1977577.8322106556,6.5600000000000005,8.165
|
||||
2020-05-07T02:46:55.724Z,13374016,98,1637768.3076169482,1976942.4981522544,8.166,6.765
|
||||
2020-05-07T02:47:02.490Z,15477914,98,2287601.832692876,1962956.753329106,6.766,7.885
|
||||
2020-05-07T02:47:10.375Z,12597108,98,1597604.0583386177,1965227.4570982838,7.885,6.41
|
||||
2020-05-07T02:47:16.786Z,13782492,98,2149819.372952738,1990251.5523465704,6.410999999999999,6.925
|
||||
2020-05-07T02:47:23.712Z,7213889,98,1041566.4164019636,1765081.7225348668,6.926,4.087
|
||||
2020-05-07T02:47:27.799Z,14092413,98,3448106.9243944217,1970966.8531468532,4.087,7.15
|
||||
2020-05-07T02:47:34.950Z,14796312,98,2069124.877639491,1963156.6936446861,7.151,7.537
|
||||
2020-05-07T02:47:42.488Z,6310417,98,837147.3865746883,1726044.0371991247,7.538,3.656
|
||||
2020-05-07T02:47:46.145Z,14718972,98,4024876.127973749,1985294.3080658214,3.657,7.414
|
||||
2020-05-07T02:47:53.560Z,14969131,98,2018763.4524612273,1975079.957778071,7.415,7.579
|
||||
2020-05-07T02:48:01.140Z,13218216,98,1743827.968337731,1994600.271616116,7.579999999999999,6.627
|
||||
2020-05-07T02:48:07.768Z,14656519,98,2211303.409776705,1973410.3945065304,6.628,7.427
|
||||
2020-05-07T02:48:15.195Z,14199628,98,1911892.8234818906,1966162.8357795624,7.427,7.222
|
||||
2020-05-07T02:48:22.418Z,6896315,98,954771.5630624394,1762411.1934577052,7.223,3.913
|
||||
2020-05-07T02:48:26.331Z,13912333,98,3555413.493483261,1961140.8232308992,3.913,7.094
|
||||
2020-05-07T02:48:33.426Z,14618759,98,2060431.148696265,1996552.7178366566,7.095,7.322
|
||||
2020-05-07T02:48:40.749Z,15481351,98,2114072.238153762,1969136.479267362,7.3229999999999995,7.862
|
||||
2020-05-07T02:48:48.612Z,14774041,98,1878931.8326338546,1988698.4789339076,7.863,7.429
|
||||
2020-05-07T02:48:56.043Z,14546853,98,1957590.230117077,2002871.1276332093,7.431,7.263
|
||||
2020-05-07T02:49:03.307Z,14626747,98,2013594.0253303964,1969932.255892256,7.264,7.425
|
||||
2020-05-07T02:49:10.732Z,14275339,98,1922604.5791245792,1980210.7088361771,7.425,7.209
|
||||
2020-05-07T02:49:17.942Z,15698454,98,2177316.7822468793,1919126.4058679708,7.21,8.18
|
||||
2020-05-07T02:49:26.123Z,13813029,98,1688427.9427942797,1998991.1722141823,8.181,6.91
|
||||
2020-05-07T02:49:33.034Z,14522315,98,2101333.38156562,1985550.3144654087,6.911,7.314
|
||||
2020-05-07T02:49:40.349Z,7565261,98,1034212.0300751879,1937823.0020491804,7.315,3.904
|
||||
2020-05-07T02:49:44.253Z,7667792,98,1964086.0655737706,1934845.3192026243,3.904,3.963
|
||||
2020-05-07T02:49:48.217Z,12688877,98,3201028.506559031,1968488.5200124108,3.9640000000000004,6.446
|
||||
2020-05-07T02:49:54.664Z,16066350,98,2492066.0772452303,1977396.923076923,6.447,8.125
|
||||
2020-05-07T02:50:02.789Z,7019375,98,863923.0769230769,1889468.3714670257,8.125,3.715
|
||||
2020-05-07T02:50:06.505Z,14776615,98,3976484.122712594,1994952.747401107,3.716,7.407
|
||||
2020-05-07T02:50:13.912Z,13984887,98,1888063.5884973675,1958393.3622741913,7.407,7.141
|
||||
2020-05-07T02:50:21.053Z,14696583,98,2058056.714745834,1993567.9598480738,7.141,7.372
|
||||
2020-05-07T02:50:28.426Z,14825128,98,2010732.1307473213,1960477.1224543771,7.373,7.562
|
||||
2020-05-07T02:50:35.988Z,14232487,98,1882106.1888389315,2007119.8702580736,7.562,7.091
|
||||
2020-05-07T02:50:43.080Z,13776030,98,1942474.6192893402,1998263.7075718015,7.092,6.894
|
||||
2020-05-07T02:50:49.974Z,14983521,98,2173414.7084421236,1981685.094564211,6.894,7.561
|
|
|
@ -0,0 +1,82 @@
|
|||
14092413
|
||||
13380122
|
||||
14387371
|
||||
14294783
|
||||
14796312
|
||||
14184248
|
||||
13626103
|
||||
13686520
|
||||
13984887
|
||||
15604551
|
||||
14275339
|
||||
14583404
|
||||
14863778
|
||||
15144990
|
||||
15481351
|
||||
14825128
|
||||
14656519
|
||||
12528177
|
||||
15698454
|
||||
14199628
|
||||
13912333
|
||||
14183601
|
||||
13724125
|
||||
14612198
|
||||
16066350
|
||||
14718972
|
||||
15554868
|
||||
15097860
|
||||
14742561
|
||||
13262441
|
||||
14626747
|
||||
15707756
|
||||
14525862
|
||||
14925983
|
||||
14969131
|
||||
14983521
|
||||
16146923
|
||||
15477914
|
||||
15567525
|
||||
15096468
|
||||
14379307
|
||||
14381752
|
||||
14546853
|
||||
14522315
|
||||
13776030
|
||||
14696583
|
||||
14776615
|
||||
13990176
|
||||
15749228
|
||||
15100468
|
||||
15285049
|
||||
14385456
|
||||
13823840
|
||||
12597108
|
||||
14107693
|
||||
15004555
|
||||
14388450
|
||||
13971118
|
||||
13782492
|
||||
13063791
|
||||
15428127
|
||||
14618759
|
||||
14774041
|
||||
13813029
|
||||
13106365
|
||||
12688877
|
||||
13374016
|
||||
14232487
|
||||
13218216
|
||||
13014161
|
||||
7667792
|
||||
7019375
|
||||
7832545
|
||||
7433139
|
||||
7565261
|
||||
6941024
|
||||
6310417
|
||||
6073357
|
||||
6567787
|
||||
7213889
|
||||
6878111
|
||||
6896315
|
|
|
@ -0,0 +1,73 @@
|
|||
{
|
||||
"collection":
|
||||
[
|
||||
{
|
||||
"id": "3badffe8-8aa9-4df6-9c79-1748858c4e52",
|
||||
"runId": "dca454f1-1a0a-4e68-8b51-cf00cb24823f",
|
||||
"name": "magenta_out",
|
||||
"timestamp": "2020-04-09T08:00:32.829Z",
|
||||
"batchId": 19,
|
||||
"numInputRows": 30080,
|
||||
"inputRowsPerSecond": 40758.807588075884,
|
||||
"processedRowsPerSecond": 40375.838926174496,
|
||||
"durationMs": {
|
||||
"addBatch": 650,
|
||||
"getBatch": 5,
|
||||
"getOffset": 22,
|
||||
"queryPlanning": 34,
|
||||
"triggerExecution": 745,
|
||||
"walCommit": 17
|
||||
},
|
||||
"stateOperators": [],
|
||||
"sources": [
|
||||
{
|
||||
"description": "FileStreamSource[file:/home/boris/yellow/magenta/raw]",
|
||||
"startOffset": {
|
||||
"logOffset": 18
|
||||
},
|
||||
"endOffset": {
|
||||
"logOffset": 19
|
||||
},
|
||||
"numInputRows": 30080,
|
||||
"inputRowsPerSecond": 40758.807588075884,
|
||||
"processedRowsPerSecond": 40375.838926174496
|
||||
}
|
||||
],
|
||||
"sink": {
|
||||
"description": "FileSink[/home/boris/yellow/magenta/processed]"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "3badffe8-8aa9-4df6-9c79-1748858c4e52",
|
||||
"runId": "dca454f1-1a0a-4e68-8b51-cf00cb24823f",
|
||||
"name": "magenta_out",
|
||||
"timestamp": "2020-04-09T08:00:33.574Z",
|
||||
"batchId": 20,
|
||||
"numInputRows": 0,
|
||||
"inputRowsPerSecond": 0.0,
|
||||
"processedRowsPerSecond": 0.0,
|
||||
"durationMs": {
|
||||
"getOffset": 2,
|
||||
"triggerExecution": 2
|
||||
},
|
||||
"stateOperators": [],
|
||||
"sources": [
|
||||
{
|
||||
"description": "FileStreamSource[file:/home/boris/yellow/magenta/raw]",
|
||||
"startOffset": {
|
||||
"logOffset": 19
|
||||
},
|
||||
"endOffset": {
|
||||
"logOffset": 19
|
||||
},
|
||||
"numInputRows": 0,
|
||||
"inputRowsPerSecond": 0.0,
|
||||
"processedRowsPerSecond": 0.0
|
||||
}
|
||||
],
|
||||
"sink": {
|
||||
"description": "FileSink[/home/boris/yellow/magenta/processed]"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
|
@ -0,0 +1,150 @@
|
|||
2020-04-10T00:30:02.276Z,30079,520,0.0,7720.482546201232,0.0,3.896
|
||||
2020-04-10T00:30:06.183Z,30079,520,7698.745840798567,17356.60703981535,3.907,1.7329999999999999
|
||||
2020-04-10T00:30:07.916Z,30076,520,17354.87593768032,18542.540073982735,1.7330000000000003,1.6220000000000003
|
||||
2020-04-10T00:30:09.539Z,30080,520,18533.579790511398,17662.947739283616,1.623,1.703
|
||||
2020-04-10T00:30:11.243Z,30082,520,17653.755868544602,19015.170670037925,1.704,1.582
|
||||
2020-04-10T00:30:12.826Z,30081,520,19002.526847757425,18001.795332136444,1.5829999999999997,1.671
|
||||
2020-04-10T00:30:14.498Z,30079,520,17989.83253588517,18717.485998755445,1.672,1.607
|
||||
2020-04-10T00:30:16.105Z,30082,520,18719.352831362787,19382.731958762884,1.6070000000000002,1.5520000000000003
|
||||
2020-04-10T00:30:17.659Z,30077,520,19354.568854568854,19658.169934640522,1.554,1.53
|
||||
2020-04-10T00:30:19.190Z,30080,520,19647.289353363816,19672.988881621975,1.531,1.529
|
||||
2020-04-10T00:30:20.720Z,30081,520,19660.78431372549,19699.410609037328,1.53,1.527
|
||||
2020-04-10T00:30:22.248Z,30075,520,19682.59162303665,18974.763406940063,1.528,1.585
|
||||
2020-04-10T00:30:23.834Z,30075,520,18962.79949558638,19217.252396166135,1.586,1.565
|
||||
2020-04-10T00:30:25.400Z,30080,520,19208.17369093231,18613.86138613861,1.566,1.6160000000000003
|
||||
2020-04-10T00:30:27.016Z,30078,520,18612.623762376235,19594.788273615635,1.616,1.535
|
||||
2020-04-10T00:30:28.551Z,30078,520,19594.788273615635,19170.17208413002,1.535,1.569
|
||||
2020-04-10T00:30:30.120Z,30079,520,19170.80943275972,19801.84331797235,1.569,1.519
|
||||
2020-04-10T00:30:31.639Z,30081,520,19803.159973666887,19520.441271901364,1.519,1.541
|
||||
2020-04-10T00:30:33.181Z,30082,520,19508.430609597923,18443.899448191292,1.5420000000000003,1.6310000000000002
|
||||
2020-04-10T00:30:34.812Z,30081,520,18443.2863274065,19622.309197651666,1.631,1.5329999999999997
|
||||
2020-04-10T00:30:36.345Z,30081,520,19622.309197651666,19908.007941760425,1.5329999999999997,1.511
|
||||
2020-04-10T00:30:37.857Z,30080,520,19894.179894179895,18894.472361809043,1.512,1.5920000000000003
|
||||
2020-04-10T00:30:39.450Z,30076,520,18880.100439422473,19416.39767591995,1.593,1.549
|
||||
2020-04-10T00:30:40.999Z,30085,520,19422.207876049066,19561.11833550065,1.549,1.538
|
||||
2020-04-10T00:30:42.537Z,30077,520,19555.91677503251,19096.507936507936,1.538,1.575
|
||||
2020-04-10T00:30:44.112Z,30080,520,19098.4126984127,19583.333333333332,1.575,1.536
|
||||
2020-04-10T00:30:45.649Z,30083,520,19572.543916720886,19935.719019218028,1.537,1.509
|
||||
2020-04-10T00:30:47.159Z,30077,520,19918.543046357616,19683.90052356021,1.51,1.528
|
||||
2020-04-10T00:30:48.687Z,30079,520,19685.20942408377,19368.319381841597,1.528,1.553
|
||||
2020-04-10T00:30:50.240Z,30078,520,19367.675466838376,19788.157894736843,1.5530000000000002,1.5199999999999998
|
||||
2020-04-10T00:30:51.761Z,30081,520,19777.120315581855,19660.78431372549,1.521,1.53
|
||||
2020-04-10T00:30:53.292Z,30083,520,19649.248856956237,18512.615384615383,1.531,1.6250000000000002
|
||||
2020-04-10T00:30:54.918Z,30083,520,18501.230012300126,19259.282970550576,1.6259999999999997,1.562
|
||||
2020-04-10T00:30:56.481Z,30082,520,19246.32117722329,19635.770234986943,1.563,1.5320000000000003
|
||||
2020-04-10T00:30:58.013Z,30079,520,19633.812010443864,18521.55172413793,1.532,1.6239999999999999
|
||||
2020-04-10T00:30:59.637Z,30080,520,18522.16748768473,19519.792342634653,1.624,1.541
|
||||
2020-04-10T00:31:01.178Z,30082,520,19521.090201168074,19648.595689092097,1.541,1.531
|
||||
2020-04-10T00:31:02.710Z,30080,520,19634.464751958225,19894.179894179895,1.532,1.512
|
||||
2020-04-10T00:31:04.223Z,30084,520,19883.674818241903,18849.624060150374,1.5130000000000001,1.596
|
||||
2020-04-10T00:31:05.820Z,30079,520,18834.690043832186,19306.161745827983,1.597,1.558
|
||||
2020-04-10T00:31:07.378Z,30078,520,19305.519897304235,19455.368693402328,1.558,1.546
|
||||
2020-04-10T00:31:08.925Z,30077,520,19442.146089204914,18497.539975399755,1.547,1.626
|
||||
2020-04-10T00:31:10.552Z,30085,520,18491.087891825446,19702.030124426983,1.627,1.527
|
||||
2020-04-10T00:31:12.079Z,30080,520,19698.755730189914,19406.451612903224,1.5270000000000001,1.5500000000000003
|
||||
2020-04-10T00:31:13.629Z,30080,520,19406.451612903224,19570.59206245934,1.5500000000000003,1.537
|
||||
2020-04-10T00:31:15.166Z,30082,520,19571.893298633702,19332.904884318767,1.537,1.556
|
||||
2020-04-10T00:31:16.722Z,30077,520,19329.69151670951,18184.401451027814,1.556,1.6539999999999997
|
||||
2020-04-10T00:31:18.376Z,30078,520,18185.006045949216,19892.85714285714,1.654,1.512
|
||||
2020-04-10T00:31:19.888Z,30084,520,19896.825396825396,19923.17880794702,1.512,1.51
|
||||
2020-04-10T00:31:21.398Z,30078,520,19919.205298013247,19330.334190231362,1.5099999999999998,1.556
|
||||
2020-04-10T00:31:22.955Z,30079,520,19318.561335902377,18601.7316017316,1.557,1.617
|
||||
2020-04-10T00:31:24.572Z,30085,520,18605.442176870747,18445.738810545678,1.6170000000000002,1.631
|
||||
2020-04-10T00:31:26.204Z,30084,520,18433.823529411766,19309.370988446724,1.632,1.5580000000000003
|
||||
2020-04-10T00:31:27.762Z,30081,520,19307.44544287548,19270.339525944906,1.558,1.5610000000000002
|
||||
2020-04-10T00:31:29.323Z,30079,520,19269.058295964125,19293.77806286081,1.561,1.559
|
||||
2020-04-10T00:31:30.882Z,30082,520,19295.702373316228,18696.08452454941,1.559,1.609
|
||||
2020-04-10T00:31:32.491Z,30083,520,18696.706028589186,19909.331568497684,1.609,1.511
|
||||
2020-04-10T00:31:34.002Z,30077,520,19905.360688285906,19454.721862871927,1.511,1.546
|
||||
2020-04-10T00:31:35.548Z,30080,520,19456.66235446313,19711.664482306685,1.546,1.5259999999999998
|
||||
2020-04-10T00:31:37.074Z,30084,520,19714.285714285714,19844.327176781004,1.526,1.516
|
||||
2020-04-10T00:31:38.591Z,30083,520,19830.586684245223,19585.286458333332,1.517,1.536
|
||||
2020-04-10T00:31:40.128Z,30076,520,19567.989590110607,19267.136450992955,1.537,1.5609999999999997
|
||||
2020-04-10T00:31:41.690Z,30075,520,19254.16133162612,19580.078125,1.562,1.536
|
||||
2020-04-10T00:31:43.227Z,30082,520,19571.893298633702,19407.74193548387,1.537,1.55
|
||||
2020-04-10T00:31:44.777Z,30081,520,19407.09677419355,19596.74267100977,1.55,1.5350000000000001
|
||||
2020-04-10T00:31:46.312Z,30083,520,19598.045602605864,19896.164021164022,1.535,1.512
|
||||
2020-04-10T00:31:47.825Z,30083,520,19883.01387970919,19948.93899204244,1.513,1.508
|
||||
2020-04-10T00:31:49.333Z,30078,520,19945.623342175066,19633.159268929503,1.508,1.532
|
||||
2020-04-10T00:31:50.865Z,30078,520,19633.159268929503,19998.67021276596,1.532,1.5039999999999998
|
||||
2020-04-10T00:31:52.370Z,30081,520,19987.375415282393,19444.731738849387,1.505,1.547
|
||||
2020-04-10T00:31:53.917Z,30078,520,19442.792501616033,19801.184990125083,1.5469999999999997,1.519
|
||||
2020-04-10T00:31:55.436Z,30081,520,19803.159973666887,19469.90291262136,1.519,1.545
|
||||
2020-04-10T00:31:56.981Z,30077,520,19467.313915857605,20037.974683544304,1.545,1.501
|
||||
2020-04-10T00:31:58.482Z,30082,520,20041.30579613591,19246.32117722329,1.501,1.563
|
||||
2020-04-10T00:32:00.045Z,30078,520,19243.76199616123,19293.136626042335,1.563,1.559
|
||||
2020-04-10T00:32:01.605Z,30082,520,19283.333333333332,19725.90163934426,1.56,1.5250000000000001
|
||||
2020-04-10T00:32:03.130Z,30079,520,19723.93442622951,19646.636185499676,1.525,1.531
|
||||
2020-04-10T00:32:04.661Z,30078,520,19645.98301763553,19417.688831504198,1.5310000000000001,1.549
|
||||
2020-04-10T00:32:06.210Z,30083,520,19420.91672046482,19383.376288659794,1.5489999999999997,1.552
|
||||
2020-04-10T00:32:07.762Z,30077,520,19379.51030927835,19555.91677503251,1.552,1.538
|
||||
2020-04-10T00:32:09.300Z,30078,520,19556.566970091026,19417.688831504198,1.538,1.549
|
||||
2020-04-10T00:32:10.850Z,30080,520,19406.451612903224,19444.085326438268,1.5500000000000003,1.547
|
||||
2020-04-10T00:32:12.398Z,30079,520,19430.878552971575,19269.058295964125,1.548,1.561
|
||||
2020-04-10T00:32:13.959Z,30079,520,19269.058295964125,19380.798969072166,1.561,1.5519999999999998
|
||||
2020-04-10T00:32:15.511Z,30079,520,19380.798969072166,19368.319381841597,1.5519999999999998,1.553
|
||||
2020-04-10T00:32:17.064Z,30076,520,19366.38763683194,19291.85375240539,1.553,1.559
|
||||
2020-04-10T00:32:18.624Z,30083,520,19283.97435897436,19700.720366732156,1.56,1.527
|
||||
2020-04-10T00:32:20.151Z,30080,520,19698.755730189914,19570.59206245934,1.5270000000000001,1.537
|
||||
2020-04-10T00:32:21.688Z,30082,520,19571.893298633702,19882.352941176472,1.537,1.513
|
||||
2020-04-10T00:32:23.202Z,30079,520,19867.239101717307,19456.01552393273,1.5139999999999998,1.546
|
||||
2020-04-10T00:32:24.748Z,30079,520,19456.01552393273,19146.40356460853,1.546,1.5710000000000002
|
||||
2020-04-10T00:32:26.319Z,30082,520,19148.313176320815,19764.78318002628,1.571,1.5220000000000002
|
||||
2020-04-10T00:32:27.842Z,30079,520,19749.83585029547,19711.009174311926,1.5230000000000001,1.526
|
||||
2020-04-10T00:32:29.369Z,30082,520,19700.06548788474,19661.437908496733,1.527,1.5299999999999998
|
||||
2020-04-10T00:32:30.899Z,30078,520,19658.823529411766,19531.16883116883,1.53,1.54
|
||||
2020-04-10T00:32:32.440Z,30082,520,19521.090201168074,19661.437908496733,1.541,1.5299999999999998
|
||||
2020-04-10T00:32:33.971Z,30076,520,19644.67668190725,19970.78353253652,1.531,1.506
|
||||
2020-04-10T00:32:35.477Z,30080,520,19973.4395750332,19481.865284974094,1.506,1.544
|
||||
2020-04-10T00:32:37.022Z,30078,520,19467.961165048546,19380.154639175256,1.545,1.552
|
||||
2020-04-10T00:32:38.574Z,30081,520,19382.08762886598,19533.116883116883,1.552,1.54
|
||||
2020-04-10T00:32:40.114Z,30080,520,19532.467532467534,19545.159194282,1.5399999999999998,1.539
|
||||
2020-04-10T00:32:41.653Z,30081,520,19545.8089668616,19295.060936497757,1.5389999999999997,1.5589999999999997
|
||||
2020-04-10T00:32:43.212Z,30083,520,19296.343810134702,19395.873629916183,1.559,1.551
|
||||
2020-04-10T00:32:44.764Z,30078,520,19380.154639175256,19658.823529411766,1.552,1.53
|
||||
2020-04-10T00:32:46.295Z,30080,520,19647.289353363816,18811.757348342715,1.531,1.599
|
||||
2020-04-10T00:32:47.895Z,30082,520,18801.25,19407.74193548387,1.6,1.55
|
||||
2020-04-10T00:32:49.446Z,30078,520,19392.649903288202,19958.858659588586,1.551,1.5070000000000001
|
||||
2020-04-10T00:32:50.953Z,30084,520,19962.8400796284,19624.26614481409,1.5070000000000001,1.533
|
||||
2020-04-10T00:32:52.486Z,30081,520,19622.309197651666,18637.546468401484,1.5329999999999997,1.6140000000000003
|
||||
2020-04-10T00:32:54.101Z,30082,520,18626.625386996904,19123.96694214876,1.615,1.573
|
||||
2020-04-10T00:32:55.675Z,30080,520,19110.54637865311,19570.59206245934,1.5740000000000003,1.537
|
||||
2020-04-10T00:32:57.212Z,30080,520,19570.59206245934,19596.091205211727,1.537,1.535
|
||||
2020-04-10T00:32:58.747Z,30079,520,19595.439739413683,19134.223918575062,1.535,1.572
|
||||
2020-04-10T00:33:00.320Z,30080,520,19122.69548633185,19557.867360208063,1.573,1.5379999999999998
|
||||
2020-04-10T00:33:01.858Z,30075,520,19554.616384915473,18974.763406940063,1.5380000000000003,1.585
|
||||
2020-04-10T00:33:03.444Z,30086,520,18969.735182849938,19460.543337645537,1.5859999999999999,1.546
|
||||
2020-04-10T00:33:04.991Z,30083,520,19446.024563671624,18825.406758448058,1.547,1.598
|
||||
2020-04-10T00:33:06.590Z,30078,520,18810.506566604126,20092.184368737475,1.5990000000000002,1.497
|
||||
2020-04-10T00:33:08.087Z,30083,520,20095.524382097527,18210.048426150122,1.497,1.652
|
||||
2020-04-10T00:33:09.739Z,30083,520,18210.048426150122,19509.079118028534,1.652,1.542
|
||||
2020-04-10T00:33:11.282Z,30079,520,19493.843162670124,19659.477124183006,1.543,1.53
|
||||
2020-04-10T00:33:12.813Z,30081,520,19647.942521227957,19520.441271901364,1.531,1.541
|
||||
2020-04-10T00:33:14.354Z,30077,520,19517.845554834523,19132.95165394402,1.541,1.572
|
||||
2020-04-10T00:33:15.926Z,30080,520,19134.860050890584,20174.379610999327,1.572,1.491
|
||||
2020-04-10T00:33:17.418Z,30082,520,20162.198391420912,19382.731958762884,1.492,1.5520000000000003
|
||||
2020-04-10T00:33:18.971Z,30079,520,19368.319381841597,19506.485084306096,1.553,1.542
|
||||
2020-04-10T00:33:20.513Z,30077,520,19505.188067444877,19787.5,1.542,1.52
|
||||
2020-04-10T00:33:22.033Z,30079,520,19788.815789473683,19025.300442757747,1.52,1.5810000000000002
|
||||
2020-04-10T00:33:23.614Z,30083,520,19027.830487033523,19383.376288659794,1.581,1.552
|
||||
2020-04-10T00:33:25.166Z,30079,520,19380.798969072166,19557.217165149545,1.5519999999999998,1.538
|
||||
2020-04-10T00:33:26.705Z,30078,520,19543.85964912281,19892.85714285714,1.539,1.512
|
||||
2020-04-10T00:33:28.217Z,30078,520,19892.85714285714,19367.675466838376,1.512,1.5530000000000002
|
||||
2020-04-10T00:33:29.770Z,30083,520,19370.895041854477,19598.045602605864,1.553,1.535
|
||||
2020-04-10T00:33:31.305Z,30077,520,19594.13680781759,19774.49046679816,1.535,1.521
|
||||
2020-04-10T00:33:32.826Z,30083,520,19778.435239973704,19948.93899204244,1.521,1.508
|
||||
2020-04-10T00:33:34.334Z,30083,520,19948.93899204244,20381.436314363145,1.508,1.476
|
||||
2020-04-10T00:33:35.810Z,30078,520,20378.048780487807,19582.03125,1.476,1.536
|
||||
2020-04-10T00:33:37.346Z,30080,520,19583.333333333332,19881.031064111037,1.536,1.5130000000000001
|
||||
2020-04-10T00:33:38.860Z,30081,520,19868.560105680317,19147.676639083387,1.514,1.571
|
||||
2020-04-10T00:33:40.431Z,30080,520,19147.04010184596,19257.362355953905,1.571,1.562
|
||||
2020-04-10T00:33:41.993Z,30081,520,19258.002560819463,19868.560105680317,1.562,1.514
|
||||
2020-04-10T00:33:43.508Z,30079,520,19854.125412541256,19788.815789473683,1.515,1.52
|
||||
2020-04-10T00:33:45.029Z,30084,520,19779.092702169626,19976.09561752988,1.521,1.506
|
||||
2020-04-10T00:33:46.535Z,30079,520,19972.775564409032,19569.941444372154,1.5059999999999998,1.537
|
||||
2020-04-10T00:33:48.073Z,30081,520,19558.51755526658,19881.692002643755,1.538,1.513
|
||||
2020-04-10T00:33:49.586Z,30081,520,19881.692002643755,19868.560105680317,1.513,1.514
|
||||
2020-04-10T00:33:51.101Z,30076,520,19852.145214521453,19747.866053841106,1.515,1.5229999999999997
|
||||
2020-04-10T00:33:52.624Z,30081,520,19751.149047931714,20054.0,1.523,1.5
|
||||
2020-04-10T00:33:54.124Z,30081,520,20054.0,19934.393638170975,1.5,1.509
|
||||
2020-04-10T00:33:55.634Z,30078,520,19919.205298013247,20159.517426273458,1.5099999999999998,1.492
|
|
|
@ -0,0 +1,106 @@
|
|||
import org.apache.spark.sql.types._
|
||||
|
||||
object Schemas {
|
||||
def getTaxiSchema() = {
|
||||
val schema_fields =
|
||||
List(
|
||||
StructField("VendorID", IntegerType),
|
||||
StructField("tpep_pickup_datetime", TimestampType),
|
||||
StructField("tpep_dropoff_datetime", TimestampType),
|
||||
StructField("passenger_count", IntegerType),
|
||||
StructField("trip_distance", DoubleType),
|
||||
StructField("RatecodeID", IntegerType),
|
||||
StructField("store_and_fwd_flag", StringType),
|
||||
StructField("PULocationID", IntegerType),
|
||||
StructField("DOLocationID", IntegerType),
|
||||
StructField("payment_type", IntegerType),
|
||||
StructField("fare_amount", DoubleType),
|
||||
StructField("extra", DoubleType),
|
||||
StructField("mta_tax", DoubleType),
|
||||
StructField("tip_amount", DoubleType),
|
||||
StructField("tolls_amount", DoubleType),
|
||||
StructField("improvement_surcharge", DoubleType),
|
||||
StructField("total_amount", DoubleType),
|
||||
StructField("congestion_surcharge", DoubleType)
|
||||
)
|
||||
StructType(schema_fields)
|
||||
}
|
||||
|
||||
def getMagentaSchema() = {
|
||||
val schema_fields =
|
||||
List(
|
||||
StructField("Text01", StringType),
|
||||
StructField("Text02", StringType),
|
||||
StructField("Text03", StringType),
|
||||
StructField("Text04", StringType),
|
||||
StructField("Text05", StringType),
|
||||
StructField("Text06", StringType),
|
||||
StructField("IntWritable01", IntegerType),
|
||||
StructField("Text07", StringType),
|
||||
StructField("Text08", StringType),
|
||||
StructField("Text09", StringType),
|
||||
StructField("Text10", StringType),
|
||||
StructField("Text11", StringType),
|
||||
StructField("Text12", StringType),
|
||||
StructField("Text13", StringType),
|
||||
StructField("LongWritable01", LongType),
|
||||
StructField("LongWritable02", LongType),
|
||||
StructField("LongWritable03", LongType),
|
||||
StructField("LongWritable04", LongType),
|
||||
StructField("LongWritable05", LongType),
|
||||
StructField("LongWritable06", LongType),
|
||||
StructField("Text14", StringType),
|
||||
StructField("Text15", StringType),
|
||||
StructField("IntWritable02", IntegerType),
|
||||
StructField("Text16", StringType),
|
||||
StructField("Text17", StringType),
|
||||
StructField("Text18", StringType),
|
||||
StructField("Text19", StringType),
|
||||
StructField("Text20", StringType),
|
||||
StructField("IntWritable03", IntegerType),
|
||||
StructField("IntWritable04", IntegerType),
|
||||
StructField("Text21", StringType),
|
||||
StructField("IntWritable05", IntegerType),
|
||||
StructField("Text22", StringType),
|
||||
StructField("Text23", StringType),
|
||||
StructField("Text24", StringType),
|
||||
StructField("Text25", StringType),
|
||||
StructField("Text26", StringType),
|
||||
StructField("IntWritable06", IntegerType),
|
||||
StructField("IntWritable07", StringType),
|
||||
StructField("Text27", StringType),
|
||||
StructField("IntWritable08", IntegerType),
|
||||
StructField("Text28", StringType),
|
||||
StructField("IntWritable09", IntegerType),
|
||||
StructField("IntWritable10", IntegerType),
|
||||
StructField("IntWritable11", IntegerType),
|
||||
StructField("Text29", StringType),
|
||||
StructField("Text30", StringType),
|
||||
StructField("Text31", StringType)
|
||||
)
|
||||
|
||||
StructType(schema_fields)
|
||||
}
|
||||
|
||||
def getMonitorSchema() = {
|
||||
val schema_fields = List(
|
||||
StructField("timestamp", DateType),
|
||||
StructField("numRows", LongType),
|
||||
StructField("avgRowSize", IntegerType),
|
||||
StructField("inputRowsPerSec", DoubleType),
|
||||
StructField("processedRowsPerSec", DoubleType),
|
||||
StructField("inputDurationSec", DoubleType),
|
||||
StructField("processedDurationSec", DoubleType)
|
||||
)
|
||||
}
|
||||
|
||||
def getQ0Schema() = {
|
||||
val schema_fields = List(
|
||||
StructField("passenger_count", IntegerType),
|
||||
StructField("year", IntegerType),
|
||||
StructField("total", IntegerType)
|
||||
)
|
||||
|
||||
StructType(schema_fields)
|
||||
}
|
||||
}
|
|
@ -0,0 +1,82 @@
|
|||
2020-05-07T02:45:07.525Z,15097860,98,0.0,1477430.276935121,0.0,10.219
|
||||
2020-05-07T02:45:17.757Z,14612198,98,1428088.1548084442,1891791.5587778352,10.232,7.724
|
||||
2020-05-07T02:45:25.482Z,6567787,98,850198.9644012946,1839201.0641276953,7.724999999999999,3.571
|
||||
2020-05-07T02:45:29.054Z,12528177,98,3507328.387458007,1949607.3762838468,3.572,6.426
|
||||
2020-05-07T02:45:35.481Z,13724125,98,2135385.8721020697,1890896.2524111324,6.427,7.258
|
||||
2020-05-07T02:45:42.740Z,13686520,98,1885455.2968728475,1951314.5138294839,7.259,7.014
|
||||
2020-05-07T02:45:49.755Z,14388450,98,2051097.647897363,1005763.3160911506,7.015,14.306
|
||||
2020-05-07T02:46:04.062Z,14379307,98,1005053.9596001956,1966266.5116915083,14.307,7.313
|
||||
2020-05-07T02:46:11.376Z,14184248,98,1939328.411266065,1951065.7496561212,7.314,7.27
|
||||
2020-05-07T02:46:18.646Z,14387371,98,1979005.6396148556,1973034.9698299507,7.27,7.292
|
||||
2020-05-07T02:50:57.535Z,15707756,98,2077470.7049332098,1972345.0527373177,7.561,7.964
|
||||
2020-05-07T02:51:05.500Z,13380122,98,1679864.6578782173,1965641.5454679006,7.965,6.807
|
||||
2020-05-07T02:51:12.307Z,6878111,98,1010446.745996768,1895318.5450537337,6.807,3.629
|
||||
2020-05-07T02:51:15.936Z,14742561,98,4062430.6971617527,1963841.8809111498,3.629,7.507
|
||||
2020-05-07T02:51:23.444Z,15554868,98,2071772.5093233883,1969718.6273268329,7.508,7.897
|
||||
2020-05-07T02:51:31.342Z,14294783,98,1809924.4112433528,1953639.8797321306,7.898,7.317
|
||||
2020-05-07T02:51:38.659Z,14183601,98,1938444.8544485443,1972958.8259841425,7.317,7.189
|
||||
2020-05-07T02:51:45.848Z,15567525,98,2165464.598692447,1950084.5546786922,7.188999999999999,7.983
|
||||
2020-05-07T02:51:53.831Z,13971118,98,1750108.7310534888,1959208.8066189873,7.983,7.131
|
||||
2020-05-07T02:52:00.962Z,7832545,98,1098379.6101528537,1959605.9544658496,7.131,3.9969999999999994
|
||||
2020-05-07T02:52:04.960Z,6073357,98,1519098.7993996998,1859001.2243648607,3.998,3.267
|
||||
2020-05-07T02:52:08.228Z,14525862,98,4444878.212974297,1961628.8993923024,3.2679999999999993,7.405
|
||||
2020-05-07T02:52:15.634Z,14107693,98,1904900.4860923577,1982531.3378302418,7.406,7.116
|
||||
2020-05-07T02:52:22.751Z,15604551,98,2192574.2588169174,1967538.8979952086,7.116999999999999,7.931
|
||||
2020-05-07T02:52:30.684Z,14863778,98,1873664.1875709065,1964808.7243886318,7.933,7.565
|
||||
2020-05-07T02:52:38.249Z,13262441,98,1753131.658955717,1956689.436411921,7.565,6.778
|
||||
2020-05-07T02:52:45.027Z,7433139,98,1096656.68338743,1940767.362924282,6.778,3.83
|
||||
2020-05-07T02:52:48.857Z,13990176,98,3652787.467362924,1976292.6967085747,3.83,7.079
|
||||
2020-05-07T02:52:55.937Z,13626103,98,1924590.8192090394,1973653.3893395134,7.08,6.904
|
||||
2020-05-07T02:53:02.841Z,13823840,98,2002294.3221320973,1979358.5337915234,6.904,6.984
|
||||
2020-05-07T02:53:09.825Z,15096468,98,2161579.0378006874,1951708.8558500323,6.984,7.735
|
||||
2020-05-07T02:53:17.561Z,6941024,98,897236.8148914168,1910548.8576933665,7.736,3.633
|
||||
2020-05-07T02:53:21.195Z,13106365,98,3606594.6615299946,1979215.493808517,3.634,6.622
|
||||
2020-05-07T02:53:27.818Z,15100468,98,2280004.2276913784,2011786.3042899014,6.623,7.506
|
||||
2020-05-07T02:53:35.324Z,13063791,98,1740446.4428457234,1980862.926459439,7.506,6.595
|
||||
2020-05-07T02:53:41.919Z,15285049,98,2317672.327520849,1998306.8374950974,6.595,7.649
|
||||
2020-05-07T02:53:49.568Z,14381752,98,1880213.3612236893,1949539.3791514167,7.649,7.377
|
||||
2020-05-07T02:53:56.946Z,14385456,98,1949777.1753862835,1999090.6058921623,7.378,7.196
|
||||
2020-05-07T02:54:04.143Z,15749228,98,2188304.5713491733,1998886.6607437492,7.197,7.879
|
||||
2020-05-07T02:54:12.022Z,15428127,98,1958132.631044549,2013064.5876826723,7.879,7.664
|
||||
2020-05-07T02:54:19.687Z,15144990,98,1975863.01369863,1992761.8421052631,7.665,7.6
|
||||
2020-05-07T02:54:27.288Z,14583404,98,1918616.497829233,1969665.586169638,7.601,7.404
|
||||
2020-05-07T02:46:25.939Z,15004555,98,2057391.33415604,1990786.1217991244,7.293,7.537
|
||||
2020-05-07T02:46:33.477Z,14925983,98,1980098.56725922,1984838.1648936172,7.538,7.52
|
||||
2020-05-07T02:46:40.998Z,13014161,98,1730376.4127110757,1984168.4708034762,7.521,6.559
|
||||
2020-05-07T02:46:47.558Z,16146923,98,2461421.18902439,1977577.8322106556,6.5600000000000005,8.165
|
||||
2020-05-07T02:46:55.724Z,13374016,98,1637768.3076169482,1976942.4981522544,8.166,6.765
|
||||
2020-05-07T02:47:02.490Z,15477914,98,2287601.832692876,1962956.753329106,6.766,7.885
|
||||
2020-05-07T02:47:10.375Z,12597108,98,1597604.0583386177,1965227.4570982838,7.885,6.41
|
||||
2020-05-07T02:47:16.786Z,13782492,98,2149819.372952738,1990251.5523465704,6.410999999999999,6.925
|
||||
2020-05-07T02:47:23.712Z,7213889,98,1041566.4164019636,1765081.7225348668,6.926,4.087
|
||||
2020-05-07T02:47:27.799Z,14092413,98,3448106.9243944217,1970966.8531468532,4.087,7.15
|
||||
2020-05-07T02:47:34.950Z,14796312,98,2069124.877639491,1963156.6936446861,7.151,7.537
|
||||
2020-05-07T02:47:42.488Z,6310417,98,837147.3865746883,1726044.0371991247,7.538,3.656
|
||||
2020-05-07T02:47:46.145Z,14718972,98,4024876.127973749,1985294.3080658214,3.657,7.414
|
||||
2020-05-07T02:47:53.560Z,14969131,98,2018763.4524612273,1975079.957778071,7.415,7.579
|
||||
2020-05-07T02:48:01.140Z,13218216,98,1743827.968337731,1994600.271616116,7.579999999999999,6.627
|
||||
2020-05-07T02:48:07.768Z,14656519,98,2211303.409776705,1973410.3945065304,6.628,7.427
|
||||
2020-05-07T02:48:15.195Z,14199628,98,1911892.8234818906,1966162.8357795624,7.427,7.222
|
||||
2020-05-07T02:48:22.418Z,6896315,98,954771.5630624394,1762411.1934577052,7.223,3.913
|
||||
2020-05-07T02:48:26.331Z,13912333,98,3555413.493483261,1961140.8232308992,3.913,7.094
|
||||
2020-05-07T02:48:33.426Z,14618759,98,2060431.148696265,1996552.7178366566,7.095,7.322
|
||||
2020-05-07T02:48:40.749Z,15481351,98,2114072.238153762,1969136.479267362,7.3229999999999995,7.862
|
||||
2020-05-07T02:48:48.612Z,14774041,98,1878931.8326338546,1988698.4789339076,7.863,7.429
|
||||
2020-05-07T02:48:56.043Z,14546853,98,1957590.230117077,2002871.1276332093,7.431,7.263
|
||||
2020-05-07T02:49:03.307Z,14626747,98,2013594.0253303964,1969932.255892256,7.264,7.425
|
||||
2020-05-07T02:49:10.732Z,14275339,98,1922604.5791245792,1980210.7088361771,7.425,7.209
|
||||
2020-05-07T02:49:17.942Z,15698454,98,2177316.7822468793,1919126.4058679708,7.21,8.18
|
||||
2020-05-07T02:49:26.123Z,13813029,98,1688427.9427942797,1998991.1722141823,8.181,6.91
|
||||
2020-05-07T02:49:33.034Z,14522315,98,2101333.38156562,1985550.3144654087,6.911,7.314
|
||||
2020-05-07T02:49:40.349Z,7565261,98,1034212.0300751879,1937823.0020491804,7.315,3.904
|
||||
2020-05-07T02:49:44.253Z,7667792,98,1964086.0655737706,1934845.3192026243,3.904,3.963
|
||||
2020-05-07T02:49:48.217Z,12688877,98,3201028.506559031,1968488.5200124108,3.9640000000000004,6.446
|
||||
2020-05-07T02:49:54.664Z,16066350,98,2492066.0772452303,1977396.923076923,6.447,8.125
|
||||
2020-05-07T02:50:02.789Z,7019375,98,863923.0769230769,1889468.3714670257,8.125,3.715
|
||||
2020-05-07T02:50:06.505Z,14776615,98,3976484.122712594,1994952.747401107,3.716,7.407
|
||||
2020-05-07T02:50:13.912Z,13984887,98,1888063.5884973675,1958393.3622741913,7.407,7.141
|
||||
2020-05-07T02:50:21.053Z,14696583,98,2058056.714745834,1993567.9598480738,7.141,7.372
|
||||
2020-05-07T02:50:28.426Z,14825128,98,2010732.1307473213,1960477.1224543771,7.373,7.562
|
||||
2020-05-07T02:50:35.988Z,14232487,98,1882106.1888389315,2007119.8702580736,7.562,7.091
|
||||
2020-05-07T02:50:43.080Z,13776030,98,1942474.6192893402,1998263.7075718015,7.092,6.894
|
||||
2020-05-07T02:50:49.974Z,14983521,98,2173414.7084421236,1981685.094564211,6.894,7.561
|
|
|
@ -0,0 +1,82 @@
|
|||
2020-05-07T03:12:19.244Z,15097860,98,0.0,1908946.7695030977,0.0,7.909
|
||||
2020-05-07T03:12:27.166Z,14612198,98,1844508.709921737,3255837.3440285204,7.922,4.488
|
||||
2020-05-07T03:12:31.655Z,6567787,98,1463084.6513700157,2127563.0061548427,4.489,3.087
|
||||
2020-05-07T03:12:34.743Z,12528177,98,4057052.1373056993,3090324.8643315244,3.088,4.054
|
||||
2020-05-07T03:12:38.798Z,13724125,98,3384494.4512946983,3369537.196169899,4.055,4.073
|
||||
2020-05-07T03:12:42.872Z,13686520,98,3359479.6269023074,3400377.6397515526,4.074,4.025
|
||||
2020-05-07T03:12:46.898Z,14388450,98,3573882.265275708,3172756.3395810365,4.026,4.535
|
||||
2020-05-07T03:12:51.434Z,14379307,98,3170041.2257495592,3460723.706377858,4.536,4.155
|
||||
2020-05-07T03:12:55.590Z,14184248,98,3412956.689124158,3294065.954482118,4.156,4.306
|
||||
2020-05-07T03:12:59.897Z,14387371,98,3340462.27072208,3543687.4384236457,4.307,4.06
|
||||
2020-05-07T03:17:22.464Z,15707756,98,2212983.3755987603,2068715.395759252,7.098,7.593
|
||||
2020-05-07T03:17:30.058Z,13380122,98,1761933.3684487753,2038720.4022550664,7.594,6.563
|
||||
2020-05-07T03:17:36.621Z,6878111,98,1048013.2561328661,1598074.1171003717,6.563,4.304
|
||||
2020-05-07T03:17:40.927Z,14742561,98,3423725.2670692056,2120926.629261977,4.306,6.950999999999999
|
||||
2020-05-07T03:17:47.878Z,15554868,98,2237788.519637462,2094931.7171717172,6.951,7.425
|
||||
2020-05-07T03:17:55.303Z,14294783,98,1925223.2996632997,2051784.5557628823,7.425,6.967
|
||||
2020-05-07T03:18:02.271Z,14183601,98,2035534.0126291618,2022184.345594525,6.968,7.014
|
||||
2020-05-07T03:18:09.286Z,15567525,98,2219176.764076978,2119761.029411765,7.015,7.343999999999999
|
||||
2020-05-07T03:18:16.631Z,13971118,98,1902126.3444520081,2051860.4787780878,7.345,6.809
|
||||
2020-05-07T03:18:23.440Z,7832545,98,1150322.3674548392,1716534.0784571555,6.809,4.563
|
||||
2020-05-07T03:18:28.004Z,6073357,98,1330709.2462751972,1596151.6425755585,4.564,3.8049999999999997
|
||||
2020-05-07T03:18:31.810Z,14525862,98,3816569.1014188123,2041869.8341298848,3.806,7.114
|
||||
2020-05-07T03:18:38.924Z,14107693,98,1983088.6983412988,2057114.7564887723,7.114,6.858
|
||||
2020-05-07T03:18:45.783Z,15604551,98,2275047.528794285,2136146.611909651,6.859,7.304999999999999
|
||||
2020-05-07T03:18:53.088Z,14863778,98,2034740.3148528405,2082636.6820793052,7.305,7.137
|
||||
2020-05-07T03:19:00.225Z,13262441,98,1858265.5177245343,1977697.7333730985,7.137,6.706
|
||||
2020-05-07T03:19:06.931Z,7433139,98,1108431.1064718163,1740374.3853898384,6.7059999999999995,4.271
|
||||
2020-05-07T03:19:11.202Z,13990176,98,3275620.697728869,2037008.7361677343,4.271,6.868
|
||||
2020-05-07T03:19:18.070Z,13626103,98,1983998.6895748398,2066753.0714394054,6.868,6.593
|
||||
2020-05-07T03:19:24.663Z,13823840,98,2096745.0326103442,2003745.470358023,6.593,6.899
|
||||
2020-05-07T03:19:31.562Z,15096468,98,2188211.045078997,2084571.6652858327,6.899,7.242
|
||||
2020-05-07T03:19:38.805Z,6941024,98,958307.8834736987,1713410.0222167366,7.243,4.051
|
||||
2020-05-07T03:19:42.857Z,13106365,98,3234542.201382034,2001277.2942433958,4.052,6.549
|
||||
2020-05-07T03:19:49.407Z,15100468,98,2305414.961832061,2077950.736204761,6.550000000000001,7.267
|
||||
2020-05-07T03:19:56.674Z,13063791,98,1797686.9409660106,2079890.3040917052,7.267,6.281
|
||||
2020-05-07T03:20:02.956Z,15285049,98,2433150.111429481,2101037.6632302403,6.282000000000001,7.275
|
||||
2020-05-07T03:20:10.231Z,14381752,98,1976873.1271477663,2097382.528802683,7.275,6.857000000000001
|
||||
2020-05-07T03:20:17.088Z,14385456,98,2097922.7067230567,2036446.2061155152,6.857,7.064
|
||||
2020-05-07T03:20:24.152Z,15749228,98,2229505.662514156,2119966.0788800647,7.064,7.429
|
||||
2020-05-07T03:20:31.582Z,15428127,98,2076463.930013459,2155368.3989941324,7.43,7.158
|
||||
2020-05-07T03:20:38.741Z,15144990,98,2115517.530381338,2061103.701687534,7.159,7.348
|
||||
2020-05-07T03:20:46.089Z,14583404,98,1984676.646706587,2038211.6002795247,7.348,7.155
|
||||
2020-05-07T03:13:03.958Z,15004555,98,3694793.154395469,3496750.174784432,4.061,4.291
|
||||
2020-05-07T03:13:08.249Z,14925983,98,3478439.291540433,3417903.1371651017,4.291,4.367
|
||||
2020-05-07T03:13:12.617Z,13014161,98,2979432.463369963,3210994.571922033,4.368,4.053
|
||||
2020-05-07T03:13:16.671Z,16146923,98,3982960.7794770594,3594595.503116652,4.054,4.492
|
||||
2020-05-07T03:13:21.164Z,13374016,98,2976633.8749165367,3082280.709840977,4.493,4.339
|
||||
2020-05-07T03:13:25.504Z,15477914,98,3566339.631336406,2089633.3198325909,4.34,7.407
|
||||
2020-05-07T03:13:32.912Z,12597108,98,1700473.5421166306,1997321.7060409067,7.408,6.307
|
||||
2020-05-07T03:13:39.220Z,13782492,98,2184922.6379201016,2037925.7725861305,6.308,6.763
|
||||
2020-05-07T03:13:45.984Z,7213889,98,1066512.2708456535,901736.125,6.764,8.0
|
||||
2020-05-07T03:13:53.985Z,14092413,98,1761331.4585676792,2020707.3415543446,8.001,6.974
|
||||
2020-05-07T03:14:00.960Z,14796312,98,2121335.053763441,2105337.5071143997,6.9750000000000005,7.028
|
||||
2020-05-07T03:14:07.988Z,6310417,98,897896.5566306204,944107.8695391981,7.028,6.684
|
||||
2020-05-07T03:14:14.672Z,14718972,98,2202120.287253142,2084544.9653023651,6.684,7.061
|
||||
2020-05-07T03:14:21.733Z,14969131,98,2119973.2332530804,2062431.937172775,7.061,7.258
|
||||
2020-05-07T03:14:28.992Z,13218216,98,1820941.7275106763,2075398.96373057,7.259,6.369
|
||||
2020-05-07T03:14:35.362Z,14656519,98,2300866.405023548,2053021.2914974086,6.37,7.139
|
||||
2020-05-07T03:14:42.502Z,14199628,98,1988743.4173669468,2035205.3891357316,7.14,6.977
|
||||
2020-05-07T03:14:49.480Z,6896315,98,988293.9237603898,1026237.3511904762,6.978,6.72
|
||||
2020-05-07T03:14:56.200Z,13912333,98,2070287.6488095238,2018034.9579344357,6.72,6.894
|
||||
2020-05-07T03:15:03.094Z,14618759,98,2120504.6417174353,2131014.4314868804,6.894,6.86
|
||||
2020-05-07T03:15:09.955Z,15481351,98,2256427.780206967,2092357.2104338424,6.860999999999999,7.399
|
||||
2020-05-07T03:15:17.355Z,14774041,98,1996492.027027027,2118749.605621684,7.4,6.972999999999999
|
||||
2020-05-07T03:15:24.328Z,14546853,98,2086168.5070988098,2064847.8353442159,6.973,7.045
|
||||
2020-05-07T03:15:31.374Z,14626747,98,2075893.69855237,2052876.7719298245,7.046,7.125
|
||||
2020-05-07T03:15:38.500Z,14275339,98,2003275.1894470952,2040500.142938822,7.126,6.996
|
||||
2020-05-07T03:15:45.497Z,15698454,98,2243597.8276404175,2111142.280796127,6.997,7.436
|
||||
2020-05-07T03:15:52.933Z,13813029,98,1857588.622915546,2056122.208990771,7.436,6.718
|
||||
2020-05-07T03:15:59.651Z,14522315,98,2161702.143495088,2086839.344733439,6.717999999999999,6.959
|
||||
2020-05-07T03:16:06.611Z,7565261,98,1086962.787356322,1690938.9807778273,6.959999999999999,4.474
|
||||
2020-05-07T03:16:11.085Z,7667792,98,1713856.0572194904,1783207.441860465,4.474,4.3
|
||||
2020-05-07T03:16:15.385Z,12688877,98,2950901.6279069767,2001715.8857864016,4.3,6.339
|
||||
2020-05-07T03:16:21.724Z,16066350,98,2534524.372929484,2061903.2340862423,6.339,7.792
|
||||
2020-05-07T03:16:29.516Z,7019375,98,900843.8141683779,1725086.0162202015,7.792,4.069
|
||||
2020-05-07T03:16:33.586Z,14776615,98,3630617.936117936,2096866.0422874982,4.07,7.047
|
||||
2020-05-07T03:16:40.634Z,13984887,98,1984234.8183881952,2026501.5215186207,7.048,6.901
|
||||
2020-05-07T03:16:47.535Z,14696583,98,2129630.92305463,2137684.8,6.901,6.875000000000001
|
||||
2020-05-07T03:16:54.410Z,14825128,98,2156382.2545454544,2048518.4468702502,6.875000000000001,7.237
|
||||
2020-05-07T03:17:01.647Z,14232487,98,1966628.0226613237,2070179.9272727272,7.237,6.875
|
||||
2020-05-07T03:17:08.523Z,13776030,98,2003494.7643979057,2013450.7453960832,6.876,6.842
|
||||
2020-05-07T03:17:15.366Z,14983521,98,2189612.889083735,2110949.7041420117,6.843,7.098000000000001
|
|
|
@ -0,0 +1,82 @@
|
|||
2020-05-13T21:56:26.904Z,15097860,98,0.0,808929.4899271324,0.0,18.664
|
||||
2020-05-13T21:56:45.581Z,14612198,98,782363.227499063,1077119.1213327437,18.677,13.565999999999999
|
||||
2020-05-13T21:56:59.148Z,6567787,98,484100.1695290042,871752.9864613751,13.567,7.534
|
||||
2020-05-13T21:57:06.683Z,12528177,98,1662664.499004645,940907.0221554638,7.535,13.315
|
||||
2020-05-13T21:57:19.999Z,13724125,98,1030649.21898468,1086543.0290554983,13.316,12.631
|
||||
2020-05-13T21:57:32.631Z,13686520,98,1083480.0506649779,1073200.031365169,12.632,12.753
|
||||
2020-05-13T21:57:45.385Z,14388450,98,1128151.9523286812,1094095.5060451678,12.754,13.150999999999998
|
||||
2020-05-13T21:57:58.536Z,14379307,98,1093400.2737434416,1105760.3045216857,13.151,13.004
|
||||
2020-05-13T21:58:11.541Z,14184248,98,1090676.5090349864,1067769.3465823547,13.005000000000003,13.283999999999999
|
||||
2020-05-13T21:58:24.826Z,14387371,98,1082978.6225065864,1123399.000546576,13.285,12.807
|
||||
2020-05-13T22:07:08.262Z,15707756,98,1178553.1212484995,1122624.0708976558,13.328,13.992
|
||||
2020-05-13T22:07:22.255Z,13380122,98,956201.1005502751,903695.9340807782,13.993,14.806
|
||||
2020-05-13T22:07:37.061Z,6878111,98,464548.8990949615,962377.3611305442,14.806,7.147
|
||||
2020-05-13T22:07:44.208Z,14742561,98,2062762.1379599832,1140094.424251798,7.147,12.931
|
||||
2020-05-13T22:07:57.140Z,15554868,98,1202819.9814413856,1155207.4266617156,12.932000000000002,13.465
|
||||
2020-05-13T22:08:10.606Z,14294783,98,1061546.3389276697,1133427.1328893118,13.466,12.612
|
||||
2020-05-13T22:08:23.219Z,14183601,98,1124522.3975263617,1112963.0414312617,12.613,12.744000000000002
|
||||
2020-05-13T22:08:35.964Z,15567525,98,1221461.3573950569,1162362.801463451,12.745000000000001,13.393
|
||||
2020-05-13T22:08:49.357Z,13971118,98,1043165.6835660419,1135955.6061468413,13.393,12.299
|
||||
2020-05-13T22:09:01.657Z,7832545,98,636792.2764227642,1012741.7895009051,12.3,7.734
|
||||
2020-05-13T22:09:09.392Z,6073357,98,785178.6683904331,936957.2662758408,7.735,6.482
|
||||
2020-05-13T22:09:15.875Z,14525862,98,2240608.0518278577,1134301.2650320162,6.483,12.806000000000001
|
||||
2020-05-13T22:09:28.681Z,14107693,98,1101647.1185381853,1137900.709791902,12.806,12.398
|
||||
2020-05-13T22:09:41.079Z,15604551,98,1258634.537828682,1147899.8823010153,12.398,13.593999999999998
|
||||
2020-05-13T22:09:54.674Z,14863778,98,1093326.811327694,1146807.962348584,13.595,12.961000000000002
|
||||
2020-05-13T22:10:07.635Z,13262441,98,1023257.5418563383,1088155.6448966195,12.961,12.188000000000002
|
||||
2020-05-13T22:10:19.824Z,7433139,98,609823.5294117647,1002446.2575859744,12.189,7.415
|
||||
2020-05-13T22:10:27.239Z,13990176,98,1886739.8516520567,1138523.4375,7.415,12.288
|
||||
2020-05-13T22:10:39.528Z,13626103,98,1108804.8661404508,1107453.1046814043,12.289,12.304000000000002
|
||||
2020-05-13T22:10:51.832Z,13823840,98,1123524.0572171651,1133287.4241678964,12.304,12.198
|
||||
2020-05-13T22:11:04.031Z,15096468,98,1237516.8456430854,1150119.457565138,12.199,13.126
|
||||
2020-05-13T22:11:17.158Z,6941024,98,528759.3509560447,983984.1224836971,13.127,7.054
|
||||
2020-05-13T22:11:24.213Z,13106365,98,1857741.3182140326,1085503.147258572,7.055,12.074000000000002
|
||||
2020-05-13T22:11:36.287Z,15100468,98,1250659.9304290211,1160414.0474909705,12.074,13.013000000000002
|
||||
2020-05-13T22:11:49.300Z,13063791,98,1003903.0969030969,1073001.3141683778,13.013,12.175
|
||||
2020-05-13T22:12:01.476Z,15285049,98,1255342.3948751641,1162714.8181956487,12.176000000000002,13.146
|
||||
2020-05-13T22:12:14.623Z,14381752,98,1093918.9168631628,1134475.9801214798,13.146999999999998,12.677
|
||||
2020-05-13T22:12:27.300Z,14385456,98,1134768.1628145461,1149365.292425695,12.676999999999998,12.516
|
||||
2020-05-13T22:12:39.817Z,15749228,98,1258227.051210354,1175227.8188194912,12.516999999999998,13.400999999999998
|
||||
2020-05-13T22:12:53.218Z,15428127,98,1151266.8457577792,1161581.6142147267,13.401000000000002,13.282
|
||||
2020-05-13T22:13:06.500Z,15144990,98,1140264.267429604,1152849.965745604,13.282,13.137000000000002
|
||||
2020-05-13T22:13:19.638Z,14583404,98,1110017.0497792664,1137817.2739330577,13.137999999999998,12.817
|
||||
2020-05-13T21:58:37.634Z,15004555,98,1171498.6727045598,1126721.859277615,12.807999999999998,13.317
|
||||
2020-05-13T21:58:50.952Z,14925983,98,1120737.5732091905,1119057.0550307394,13.318000000000001,13.337999999999997
|
||||
2020-05-13T21:59:04.291Z,13014161,98,975647.4248444411,1036819.7100063735,13.339,12.552
|
||||
2020-05-13T21:59:16.844Z,16146923,98,1286299.928303991,1109601.6355140188,12.553,14.551999999999998
|
||||
2020-05-13T21:59:31.397Z,13374016,98,918986.8755583041,1052078.036500944,14.552999999999999,12.712
|
||||
2020-05-13T21:59:44.109Z,15477914,98,1217582.913782253,1126403.7551852122,12.712,13.740999999999998
|
||||
2020-05-13T21:59:57.851Z,12597108,98,916686.6540532673,1010031.109685696,13.742,12.472
|
||||
2020-05-13T22:00:10.324Z,13782492,98,1104986.1300408882,1097944.0771130407,12.473,12.552999999999999
|
||||
2020-05-13T22:00:22.878Z,7213889,98,574628.7239126972,484836.95140802476,12.554,14.879
|
||||
2020-05-13T22:00:37.758Z,14092413,98,947070.7661290322,1106328.5445124824,14.88,12.738
|
||||
2020-05-13T22:00:50.497Z,14796312,98,1161497.13478295,1102967.722698472,12.739,13.415
|
||||
2020-05-13T22:01:03.912Z,6310417,98,470400.07454342156,470294.90236995084,13.415,13.418
|
||||
2020-05-13T22:01:17.330Z,14718972,98,1096957.2216425696,1102792.53764891,13.418000000000001,13.347
|
||||
2020-05-13T22:01:30.678Z,14969131,98,1121451.2286484865,1118601.9279629353,13.348,13.382
|
||||
2020-05-13T22:01:44.061Z,13218216,98,987687.0656803407,1060511.5532734275,13.383,12.463999999999999
|
||||
2020-05-13T22:01:56.525Z,14656519,98,1175908.1354300384,1078319.5261918774,12.464,13.592000000000002
|
||||
2020-05-13T22:02:10.118Z,14199628,98,1044627.97027882,1060782.0110563275,13.593,13.386000000000001
|
||||
2020-05-13T22:02:23.505Z,6896315,98,515150.14566370356,506486.1192714454,13.387,13.616
|
||||
2020-05-13T22:02:37.122Z,13912333,98,1021688.551075861,1078809.9410669974,13.617,12.896000000000003
|
||||
2020-05-13T22:02:50.019Z,14618759,98,1133500.736605412,1115084.591914569,12.897,13.11
|
||||
2020-05-13T22:03:03.129Z,15481351,98,1180881.0831426394,1133832.6497729602,13.109999999999998,13.654000000000002
|
||||
2020-05-13T22:03:16.784Z,14774041,98,1081951.0069571587,1131416.8325930464,13.654999999999998,13.058
|
||||
2020-05-13T22:03:29.842Z,14546853,98,1114018.4561188545,1119936.3307413964,13.057999999999998,12.989000000000003
|
||||
2020-05-13T22:03:42.831Z,14626747,98,1126087.2276541688,1103905.433962264,12.989,13.25
|
||||
2020-05-13T22:03:56.082Z,14275339,98,1077302.7696022943,1112392.9712460064,13.250999999999998,12.833
|
||||
2020-05-13T22:04:08.915Z,15698454,98,1223287.929556612,1090094.715644747,12.833,14.400999999999998
|
||||
2020-05-13T22:04:23.316Z,13813029,98,959171.5158669536,1107434.378257035,14.401,12.473000000000003
|
||||
2020-05-13T22:04:35.790Z,14522315,98,1164206.7500400834,1119426.1157789254,12.474,12.973
|
||||
2020-05-13T22:04:48.763Z,7565261,98,583154.3205118323,1000298.9554409626,12.973,7.563
|
||||
2020-05-13T22:04:56.327Z,7667792,98,1013721.8402961396,987354.1076487252,7.564,7.766
|
||||
2020-05-13T22:05:04.094Z,12688877,98,1633690.8716364105,1027189.9133813649,7.7669999999999995,12.353
|
||||
2020-05-13T22:05:16.448Z,16066350,98,1300497.8144730453,1128571.9303175048,12.354,14.236
|
||||
2020-05-13T22:05:30.684Z,7019375,98,493072.14105085697,952942.5739885963,14.236,7.366
|
||||
2020-05-13T22:05:38.050Z,14776615,98,2006056.8829758351,1143258.413926499,7.366,12.925
|
||||
2020-05-13T22:05:50.975Z,13984887,98,1082002.8626692456,1112206.6963575631,12.925,12.574000000000002
|
||||
2020-05-13T22:06:03.550Z,14696583,98,1168714.3538767397,1128249.8848456931,12.574999999999998,13.026000000000002
|
||||
2020-05-13T22:06:16.577Z,14825128,98,1138030.8589851847,1123200.8485491325,13.027,13.199
|
||||
2020-05-13T22:06:29.777Z,14232487,98,1078218.7121212122,1119522.300007866,13.2,12.713
|
||||
2020-05-13T22:06:42.490Z,13776030,98,1083617.5568315897,1107041.9479267115,12.713000000000001,12.444
|
||||
2020-05-13T22:06:54.934Z,14983521,98,1204075.9402121503,1124213.7605042018,12.444,13.328
|
|
|
@ -0,0 +1,82 @@
|
|||
2020-05-07T03:22:58.528Z,15097860,98,0.0,1355283.6624775582,0.0,11.14
|
||||
2020-05-07T03:23:09.683Z,14612198,98,1309923.6216943075,1979436.1961528042,11.155,7.382
|
||||
2020-05-07T03:23:17.066Z,6567787,98,889582.4190708384,851853.0479896239,7.383,7.71
|
||||
2020-05-07T03:23:24.777Z,12528177,98,1624714.952665024,1948091.5876224537,7.711,6.431
|
||||
2020-05-07T03:23:31.208Z,13724125,98,2134057.689317369,2001476.5932623595,6.431,6.857
|
||||
2020-05-07T03:23:38.066Z,13686520,98,1995701.3706620007,1989319.7674418604,6.858,6.88
|
||||
2020-05-07T03:23:44.947Z,14388450,98,2091040.5464322045,2053146.404109589,6.881,7.008
|
||||
2020-05-07T03:23:51.956Z,14379307,98,2051549.0084177486,1996017.0738478624,7.009,7.204
|
||||
2020-05-07T03:23:59.161Z,14184248,98,1968667.3143650242,2022276.5896777872,7.205,7.014
|
||||
2020-05-07T03:24:06.176Z,14387371,98,2050943.834640057,2005488.0122665178,7.015,7.174
|
||||
2020-05-07T03:28:55.403Z,15707756,98,2159735.459920253,2058413.8382911806,7.273,7.631
|
||||
2020-05-07T03:29:03.034Z,13380122,98,1753390.3813392739,1944502.5432349949,7.631,6.881
|
||||
2020-05-07T03:29:09.916Z,6878111,98,999434.9026445801,996683.2343138676,6.882,6.901
|
||||
2020-05-07T03:29:16.818Z,14742561,98,2135983.917705013,2044738.0027739252,6.902,7.21
|
||||
2020-05-07T03:29:24.028Z,15554868,98,2157401.9417475727,2066268.331562168,7.21,7.528
|
||||
2020-05-07T03:29:31.556Z,14294783,98,1898881.907545165,2019037.1468926554,7.528,7.08
|
||||
2020-05-07T03:29:38.636Z,14183601,98,2003333.4745762711,2019592.9090132422,7.08,7.023
|
||||
2020-05-07T03:29:45.660Z,15567525,98,2216333.285876993,2067674.9900385179,7.024,7.529
|
||||
2020-05-07T03:29:53.190Z,13971118,98,1855394.1567065073,2085241.4925373134,7.53,6.7
|
||||
2020-05-07T03:29:59.891Z,7832545,98,1168862.110132816,1711283.5918724055,6.701,4.577
|
||||
2020-05-07T03:30:04.469Z,6073357,98,1326639.7990388814,956735.5072463768,4.578,6.348
|
||||
2020-05-07T03:30:10.817Z,14525862,98,2288258.034026465,2007998.6176389272,6.348000000000001,7.234
|
||||
2020-05-07T03:30:18.052Z,14107693,98,1949923.013130615,2005357.9246624021,7.235,7.035
|
||||
2020-05-07T03:30:25.088Z,15604551,98,2217815.66230813,2117881.5146579803,7.035999999999999,7.368
|
||||
2020-05-07T03:30:32.457Z,14863778,98,2017068.53032976,2057555.0941306755,7.369,7.224
|
||||
2020-05-07T03:30:39.682Z,13262441,98,1835631.9723183392,1983317.0330492,7.225,6.687
|
||||
2020-05-07T03:30:46.370Z,7433139,98,1111414.3241626795,1621539.9214659687,6.688,4.584
|
||||
2020-05-07T03:30:50.955Z,13990176,98,3051292.4754634677,2079395.957193817,4.585,6.728
|
||||
2020-05-07T03:30:57.684Z,13626103,98,2024981.869519988,2020477.9062870701,6.729,6.744
|
||||
2020-05-07T03:31:04.429Z,13823840,98,2049494.4403261675,2002004.3446777696,6.745,6.905
|
||||
2020-05-07T03:31:11.335Z,15096468,98,2185993.049522155,2064897.8251949118,6.905999999999999,7.311
|
||||
2020-05-07T03:31:18.646Z,6941024,98,949394.6108603474,1594171.7960496095,7.311,4.354
|
||||
2020-05-07T03:31:23.001Z,13106365,98,3009498.277841561,1986113.805121988,4.355,6.599
|
||||
2020-05-07T03:31:29.601Z,15100468,98,2287949.6969696973,2064315.5160628844,6.599999999999999,7.315
|
||||
2020-05-07T03:31:36.917Z,13063791,98,1785646.6648441772,2035492.5210345902,7.316,6.418
|
||||
2020-05-07T03:31:43.335Z,15285049,98,2381590.6824555937,2086696.109215017,6.418,7.325
|
||||
2020-05-07T03:31:50.661Z,14381752,98,1963111.1111111112,2023319.0770962297,7.326,7.108
|
||||
2020-05-07T03:31:57.769Z,14385456,98,2023840.1800787845,2063318.4165232356,7.108,6.972
|
||||
2020-05-07T03:32:04.742Z,15749228,98,2258601.462785028,2096821.7281320728,6.973,7.511
|
||||
2020-05-07T03:32:12.253Z,15428127,98,2054070.9625882038,2090815.422143922,7.511,7.379
|
||||
2020-05-07T03:32:19.632Z,15144990,98,2052444.7757148666,2044134.1611553517,7.379,7.409
|
||||
2020-05-07T03:32:27.041Z,14583404,98,1968336.3476852477,2025472.7777777778,7.409,7.2
|
||||
2020-05-07T03:24:13.350Z,15004555,98,2091518.6785614719,2088607.3218262806,7.174,7.184
|
||||
2020-05-07T03:24:20.535Z,14925983,98,2077381.0716771053,2053657.5399009357,7.185,7.268
|
||||
2020-05-07T03:24:27.804Z,13014161,98,1790364.699408447,1998182.2508828498,7.269,6.513
|
||||
2020-05-07T03:24:34.317Z,16146923,98,2479183.632734531,2078111.0682110684,6.513,7.77
|
||||
2020-05-07T03:24:42.088Z,13374016,98,1721016.0854458886,1986043.362043362,7.771,6.734
|
||||
2020-05-07T03:24:48.823Z,15477914,98,2298131.2546399403,2025639.8377175762,6.735000000000001,7.641
|
||||
2020-05-07T03:24:56.464Z,12597108,98,1648620.3376521398,2007187.380497132,7.641,6.276
|
||||
2020-05-07T03:25:02.740Z,13782492,98,2196063.09751434,1937920.6974128233,6.276000000000001,7.112
|
||||
2020-05-07T03:25:09.853Z,7213889,98,1014183.7480669196,967528.0311158798,7.113,7.456
|
||||
2020-05-07T03:25:17.310Z,14092413,98,1889823.387421215,2002047.5919874983,7.457,7.039
|
||||
2020-05-07T03:25:24.350Z,14796312,98,2101748.8636363638,2024950.3216094156,7.039999999999999,7.307
|
||||
2020-05-07T03:25:31.658Z,6310417,98,863494.389709907,921363.2647101766,7.308,6.849
|
||||
2020-05-07T03:25:38.507Z,14718972,98,2149068.7691633813,1964363.0054717737,6.849000000000001,7.493
|
||||
2020-05-07T03:25:46.001Z,14969131,98,1997482.1190285562,1932747.708198838,7.494,7.745
|
||||
2020-05-07T03:25:53.747Z,13218216,98,1706457.0100697132,1866979.6610169492,7.746,7.08
|
||||
2020-05-07T03:26:00.828Z,14656519,98,2069837.4523372403,2074818.657984145,7.081,7.064
|
||||
2020-05-07T03:26:07.892Z,14199628,98,2010139.8640996602,2013845.979293717,7.064,7.051
|
||||
2020-05-07T03:26:14.943Z,6896315,98,978061.9770245355,1001643.427741467,7.051,6.885
|
||||
2020-05-07T03:26:21.828Z,13912333,98,2020672.9121278143,2026855.0407925409,6.885,6.864
|
||||
2020-05-07T03:26:28.692Z,14618759,98,2129772.5815850818,2075938.5117864243,6.864,7.042
|
||||
2020-05-07T03:26:35.735Z,15481351,98,2198118.8414028115,2093205.922120065,7.042999999999999,7.396
|
||||
2020-05-07T03:26:43.132Z,14774041,98,1997301.74395025,2064278.4686321085,7.397,7.157
|
||||
2020-05-07T03:26:50.289Z,14546853,98,2032535.0006986167,2083181.0110267794,7.157,6.983
|
||||
2020-05-07T03:26:57.273Z,14626747,98,2094322.3081328752,2041986.1789752897,6.984,7.163
|
||||
2020-05-07T03:27:04.436Z,14275339,98,1992927.404718693,2026595.5423055084,7.163,7.044
|
||||
2020-05-07T03:27:11.480Z,15698454,98,2228627.768313458,1176354.7396028475,7.0440000000000005,13.345
|
||||
2020-05-07T03:27:24.826Z,13813029,98,1034993.9307657725,2045465.5708573966,13.346,6.753
|
||||
2020-05-07T03:27:31.580Z,14522315,98,2150179.893396506,2069000.5698817496,6.753999999999999,7.019
|
||||
2020-05-07T03:27:38.600Z,7565261,98,1077672.507122507,1654332.1670675704,7.0200000000000005,4.573
|
||||
2020-05-07T03:27:43.174Z,7667792,98,1676386.5325754264,1760687.0264064292,4.574,4.355
|
||||
2020-05-07T03:27:47.530Z,12688877,98,2912965.3351698807,1971853.4576534578,4.356,6.435
|
||||
2020-05-07T03:27:53.966Z,16066350,98,2496325.357364823,2091155.7985162046,6.435999999999999,7.683
|
||||
2020-05-07T03:28:01.649Z,7019375,98,913624.235324743,1689786.9523350988,7.683,4.154
|
||||
2020-05-07T03:28:05.804Z,14776615,98,3556345.367027677,2028918.7148153232,4.155,7.283
|
||||
2020-05-07T03:28:13.088Z,13984887,98,1919946.046128501,2019769.9306759099,7.284,6.924
|
||||
2020-05-07T03:28:20.013Z,14696583,98,2122250.2527075815,2050018.5520993166,6.924999999999999,7.169
|
||||
2020-05-07T03:28:27.182Z,14825128,98,2067949.2258334497,2021975.9956355703,7.169,7.332
|
||||
2020-05-07T03:28:34.515Z,14232487,98,1940881.9037228965,2081685.9733801375,7.333,6.837
|
||||
2020-05-07T03:28:41.352Z,13776030,98,2014923.2119350594,2032462.3782826795,6.837,6.778
|
||||
2020-05-07T03:28:48.130Z,14983521,98,2210610.9471820598,2060440.1815181517,6.778,7.272
|
|
|
@ -0,0 +1,82 @@
|
|||
2020-05-07T03:33:50.494Z,15097860,98,0.0,799886.6225165563,0.0,18.875
|
||||
2020-05-07T03:34:09.386Z,14612198,98,773459.5596019479,1012486.0033259423,18.892,14.432
|
||||
2020-05-07T03:34:23.819Z,6567787,98,455053.48853322247,534836.0749185669,14.433,12.279999999999998
|
||||
2020-05-07T03:34:36.100Z,12528177,98,1020126.781206742,907050.1737619461,12.281,13.812
|
||||
2020-05-07T03:34:49.912Z,13724125,98,993637.778743122,1003005.5543374991,13.812,13.683
|
||||
2020-05-07T03:35:03.596Z,13686520,98,1000184.1566793335,1001721.4374588304,13.684,13.663
|
||||
2020-05-07T03:35:17.260Z,14388450,98,1053018.881733021,995740.4844290658,13.664000000000001,14.45
|
||||
2020-05-07T03:35:31.711Z,14379307,98,995038.8900422116,1016852.202814511,14.451,14.141
|
||||
2020-05-07T03:35:45.852Z,14184248,98,1003058.340994272,998820.3647630449,14.141,14.201
|
||||
2020-05-07T03:36:00.053Z,14387371,98,1013123.794099007,1003233.4565232551,14.201,14.341
|
||||
2020-05-07T03:45:22.452Z,15707756,98,1098982.4389561324,1042388.7451058464,14.293,15.069
|
||||
2020-05-07T03:45:37.523Z,13380122,98,887805.8522991175,1004287.4727914133,15.071,13.323
|
||||
2020-05-07T03:45:50.847Z,6878111,98,516219.6787751426,604562.8021446778,13.324,11.377
|
||||
2020-05-07T03:46:02.225Z,14742561,98,1295707.5936016873,1027857.5611796696,11.378000000000002,14.343
|
||||
2020-05-07T03:46:16.568Z,15554868,98,1084491.9472913616,1020995.6022317033,14.343,15.235
|
||||
2020-05-07T03:46:31.804Z,14294783,98,938224.1401942767,1035328.673861085,15.236,13.807
|
||||
2020-05-07T03:46:45.612Z,14183601,98,1027201.6946697567,1029288.8969521045,13.808,13.78
|
||||
2020-05-07T03:46:59.392Z,15567525,98,1129718.795355588,1019484.2829076621,13.779999999999998,15.27
|
||||
2020-05-07T03:47:14.662Z,13971118,98,914938.9652914212,1028346.6804063006,15.27,13.586
|
||||
2020-05-07T03:47:28.249Z,7832545,98,576473.4672849047,928246.6224223749,13.587,8.438
|
||||
2020-05-07T03:47:36.688Z,6073357,98,719677.3314373741,590334.0785381027,8.439,10.288
|
||||
2020-05-07T03:47:46.977Z,14525862,98,1411785.596267859,1024390.832157969,10.289,14.18
|
||||
2020-05-07T03:48:01.158Z,14107693,98,994830.6184331148,1020521.7737268519,14.181,13.824
|
||||
2020-05-07T03:48:14.983Z,15604551,98,1128719.7830018084,1034098.8071570577,13.824999999999998,15.09
|
||||
2020-05-07T03:48:30.074Z,14863778,98,984943.2111854749,1036670.2468963594,15.091,14.338
|
||||
2020-05-07T03:48:44.412Z,13262441,98,924985.4233505371,999505.6899540282,14.338,13.269
|
||||
2020-05-07T03:48:57.681Z,7433139,98,560188.3337101515,917444.9518637373,13.269,8.102
|
||||
2020-05-07T03:49:05.784Z,13990176,98,1726542.7619400222,1007357.1428571428,8.103,13.888
|
||||
2020-05-07T03:49:19.672Z,13626103,98,981142.2091013825,995041.8431429822,13.888,13.694
|
||||
2020-05-07T03:49:33.366Z,13823840,98,1009481.5247553672,1023230.1998519616,13.694,13.51
|
||||
2020-05-07T03:49:46.877Z,15096468,98,1117346.45844127,1039415.3125860645,13.511000000000001,14.524
|
||||
2020-05-07T03:50:01.401Z,6941024,98,477900.30294684664,883981.6607233825,14.524,7.852000000000001
|
||||
2020-05-07T03:50:09.253Z,13106365,98,1669175.369332654,1019791.86118892,7.852000000000001,12.852
|
||||
2020-05-07T03:50:22.105Z,15100468,98,1174950.8247743542,1039261.3902271164,12.852,14.53
|
||||
2020-05-07T03:50:36.635Z,13063791,98,899090.9153475568,1020847.9331093224,14.53,12.797
|
||||
2020-05-07T03:50:49.433Z,15285049,98,1194331.0673542742,1028603.5666218036,12.797999999999998,14.86
|
||||
2020-05-07T03:51:04.293Z,14381752,98,967816.4199192463,1028443.3638443936,14.86,13.984
|
||||
2020-05-07T03:51:18.277Z,14385456,98,1028708.23798627,1028634.6800143011,13.984,13.985
|
||||
2020-05-07T03:51:32.263Z,15749228,98,1126070.928070928,1040927.1645736946,13.986,15.13
|
||||
2020-05-07T03:51:47.393Z,15428127,98,1019704.3621943159,1033918.1745074387,15.13,14.922
|
||||
2020-05-07T03:52:02.315Z,15144990,98,1014943.7072778448,1017808.4677419354,14.922,14.88
|
||||
2020-05-07T03:52:17.195Z,14583404,98,980067.4731182795,1016194.2721761549,14.88,14.351
|
||||
2020-05-07T03:36:14.395Z,15004555,98,1046196.8344721795,1012521.4251973816,14.342,14.819
|
||||
2020-05-07T03:36:29.214Z,14925983,98,1007219.313044065,995862.2231118229,14.819,14.988
|
||||
2020-05-07T03:36:44.203Z,13014161,98,868247.4481286276,1002322.9359211337,14.989,12.984
|
||||
2020-05-07T03:36:57.188Z,16146923,98,1243505.8144012322,1027092.6149736022,12.985,15.721
|
||||
2020-05-07T03:37:12.909Z,13374016,98,850710.2601615674,981363.076019959,15.721,13.628
|
||||
2020-05-07T03:37:26.537Z,15477914,98,1135743.6160845319,1041021.9262846381,13.628,14.868
|
||||
2020-05-07T03:37:41.406Z,12597108,98,847206.1335664806,996685.4972703537,14.869,12.639
|
||||
2020-05-07T03:37:54.046Z,13782492,98,1090387.0253164556,1010743.0331475507,12.64,13.636
|
||||
2020-05-07T03:38:07.683Z,7213889,98,528993.8402874533,592613.8996138996,13.637,12.173
|
||||
2020-05-07T03:38:19.857Z,14092413,98,1157582.7994085758,1003518.6925870541,12.174,14.043
|
||||
2020-05-07T03:38:33.901Z,14796312,98,1053568.2141839932,1036228.8675677568,14.044,14.279
|
||||
2020-05-07T03:38:48.180Z,6310417,98,441936.9003431613,580108.2000367715,14.279,10.878
|
||||
2020-05-07T03:38:59.059Z,14718972,98,1352971.0451328247,1031245.8488054369,10.879,14.273
|
||||
2020-05-07T03:39:13.332Z,14969131,98,1048772.5775940588,1032852.4805078313,14.272999999999998,14.493
|
||||
2020-05-07T03:39:27.825Z,13218216,98,912041.3992962119,999033.78429446,14.493,13.231
|
||||
2020-05-07T03:39:41.057Z,14656519,98,1107657.1191051996,1016261.1981694633,13.232,14.422
|
||||
2020-05-07T03:39:55.479Z,14199628,98,984581.0567189016,1028511.3718673041,14.422,13.806
|
||||
2020-05-07T03:40:09.286Z,6896315,98,499479.61179112043,569990.4950822382,13.807,12.099
|
||||
2020-05-07T03:40:21.386Z,13912333,98,1149779.5867768596,1029475.5808790884,12.1,13.514
|
||||
2020-05-07T03:40:34.901Z,14618759,98,1081669.182389937,1032179.552354727,13.515,14.163
|
||||
2020-05-07T03:40:49.064Z,15481351,98,1093084.1629598248,1041463.2357887656,14.163000000000002,14.865
|
||||
2020-05-07T03:41:03.929Z,14774041,98,993880.9956273125,1013934.596115572,14.865,14.571
|
||||
2020-05-07T03:41:18.501Z,14546853,98,998274.293164974,1041888.9127632144,14.572,13.962
|
||||
2020-05-07T03:41:32.464Z,14626747,98,1047536.131203896,1036769.7051318401,13.963,14.108
|
||||
2020-05-07T03:41:46.573Z,14275339,98,1011789.566943086,1029001.5858141715,14.109,13.873
|
||||
2020-05-07T03:42:00.447Z,15698454,98,1131501.6577771371,674766.9890393295,13.873999999999999,23.265
|
||||
2020-05-07T03:42:23.712Z,13813029,98,593725.7253384913,1026838.3140053523,23.265,13.452
|
||||
2020-05-07T03:42:37.164Z,14522315,98,1079565.492120131,1032661.2387115124,13.451999999999998,14.063
|
||||
2020-05-07T03:42:51.227Z,7565261,98,537954.9882670838,939550.5464480874,14.063000000000002,8.052
|
||||
2020-05-07T03:42:59.280Z,7667792,98,952165.9009064944,905716.0406331208,8.053,8.466
|
||||
2020-05-07T03:43:07.746Z,12688877,98,1498804.2759272384,1004502.6124129196,8.466,12.632
|
||||
2020-05-07T03:43:20.379Z,16066350,98,1271776.3001662313,1035136.2669931061,12.633,15.521
|
||||
2020-05-07T03:43:35.901Z,7019375,98,452221.04110295064,896242.9775280899,15.522,7.832
|
||||
2020-05-07T03:43:43.734Z,14776615,98,1886456.657730116,1014041.6552292068,7.833,14.572
|
||||
2020-05-07T03:43:58.306Z,13984887,98,959709.5113917101,1012224.0156340474,14.572,13.816
|
||||
2020-05-07T03:44:12.122Z,14696583,98,1063736.4649681528,1027661.2125026222,13.816,14.301
|
||||
2020-05-07T03:44:26.424Z,14825128,98,1036577.2619214096,1035346.6024163698,14.302,14.319
|
||||
2020-05-07T03:44:40.743Z,14232487,98,993958.1674697953,1027839.0265039359,14.319,13.847
|
||||
2020-05-07T03:44:54.591Z,13776030,98,994802.8596187175,1015332.399764151,13.848,13.568
|
||||
2020-05-07T03:45:08.159Z,14983521,98,1104327.9038915094,1048311.8309662073,13.568,14.293
|
|
|
@ -0,0 +1,82 @@
|
|||
2020-05-07T03:57:51.948Z,15097860,98,0.0,769396.1168017122,0.0,19.623
|
||||
2020-05-07T03:58:11.585Z,14612198,98,744115.598105617,921963.404631207,19.637,15.849
|
||||
2020-05-07T03:58:27.435Z,6567787,98,414371.41955835966,507478.51954875595,15.849999999999998,12.942
|
||||
2020-05-07T03:58:40.378Z,12528177,98,967950.0115892761,871101.1681268252,12.943,14.382
|
||||
2020-05-07T03:58:54.761Z,13724125,98,954190.7112563443,905643.7244291937,14.383,15.154
|
||||
2020-05-07T03:59:09.916Z,13686520,98,903102.6064005279,898419.3251936458,15.155,15.234
|
||||
2020-05-07T03:59:25.151Z,14388450,98,944433.8693797177,878898.6622686458,15.235,16.371
|
||||
2020-05-07T03:59:41.523Z,14379307,98,878286.5257757147,901241.4290191162,16.372,15.955
|
||||
2020-05-07T03:59:57.479Z,14184248,98,888960.1403860617,880790.3626428217,15.956,16.104
|
||||
2020-05-07T04:00:13.584Z,14387371,98,893348.0906550761,897415.8557884231,16.105,16.032
|
||||
2020-05-07T04:10:44.895Z,15707756,98,953835.0740830702,918582.2222222221,16.468,17.1
|
||||
2020-05-07T04:11:01.995Z,13380122,98,782463.2748538011,857480.2614714176,17.1,15.604
|
||||
2020-05-07T04:11:17.600Z,6878111,98,440763.2809996796,530104.8940269749,15.605,12.975000000000001
|
||||
2020-05-07T04:11:30.575Z,14742561,98,1136228.2080924856,914097.2842261904,12.975,16.128
|
||||
2020-05-07T04:11:46.704Z,15554868,98,964403.7448074896,918178.8560297503,16.129,16.941
|
||||
2020-05-07T04:12:03.645Z,14294783,98,843798.0638687209,903475.0979648591,16.941,15.822
|
||||
2020-05-07T04:12:19.467Z,14183601,98,896448.0470231324,908215.4703208042,15.822,15.617
|
||||
2020-05-07T04:12:35.084Z,15567525,98,996831.977972722,919305.8344159678,15.617,16.934
|
||||
2020-05-07T04:12:52.018Z,13971118,98,825033.5419865359,898290.8763582588,16.934,15.553
|
||||
2020-05-07T04:13:07.571Z,7832545,98,503603.4848582267,746881.3769428816,15.553,10.487
|
||||
2020-05-07T04:13:18.059Z,6073357,98,579076.754385965,512952.4493243243,10.488,11.84
|
||||
2020-05-07T04:13:29.900Z,14525862,98,1226742.8426653156,910369.8922035599,11.840999999999998,15.956
|
||||
2020-05-07T04:13:45.856Z,14107693,98,884162.2587114566,901622.8670032594,15.956,15.647
|
||||
2020-05-07T04:14:01.503Z,15604551,98,997287.0837860293,900747.5756176402,15.647,17.324
|
||||
2020-05-07T04:14:18.827Z,14863778,98,857987.6471946432,900835.0303030303,17.324,16.5
|
||||
2020-05-07T04:14:35.327Z,13262441,98,803784.303030303,890216.2035172506,16.5,14.898000000000001
|
||||
2020-05-07T04:14:50.226Z,7433139,98,498901.87260889995,749006.3482466748,14.899,9.924
|
||||
2020-05-07T04:15:00.151Z,13990176,98,1409589.5214105793,899053.7883169462,9.925,15.561
|
||||
2020-05-07T04:15:15.712Z,13626103,98,875657.2842362316,894805.8182295772,15.561,15.228
|
||||
2020-05-07T04:15:30.940Z,13823840,98,907790.9114788547,892666.9249644841,15.228,15.486
|
||||
2020-05-07T04:15:46.426Z,15096468,98,974846.1836497481,912669.6088507345,15.486,16.541
|
||||
2020-05-07T04:16:02.967Z,6941024,98,419625.4156338794,703672.3438767234,16.541,9.864
|
||||
2020-05-07T04:16:12.831Z,13106365,98,1328706.914030819,884310.43789218,9.864,14.821
|
||||
2020-05-07T04:16:27.652Z,15100468,98,1018856.2175291816,918687.5950599257,14.821,16.437
|
||||
2020-05-07T04:16:44.090Z,13063791,98,794731.1716753864,886221.4910793026,16.438,14.741
|
||||
2020-05-07T04:16:58.832Z,15285049,98,1036836.8606701939,903531.8909972217,14.742,16.917
|
||||
2020-05-07T04:17:15.750Z,14381752,98,850085.8257477244,898747.1566054244,16.918,16.002
|
||||
2020-05-07T04:17:31.752Z,14385456,98,898978.6276715411,906855.9541070416,16.002,15.863
|
||||
2020-05-07T04:17:47.615Z,15749228,98,992827.838366009,928774.4294391696,15.863,16.957
|
||||
2020-05-07T04:18:04.573Z,15428127,98,909784.5854463971,921577.3848635087,16.958,16.741
|
||||
2020-05-07T04:18:21.314Z,15144990,98,904664.5959022759,915326.3628671583,16.741,16.546
|
||||
2020-05-07T04:18:37.861Z,14583404,98,881332.205233577,895621.445679543,16.547,16.283
|
||||
2020-05-07T04:00:29.617Z,15004555,98,935854.4876192851,938312.4882746545,16.033,15.991
|
||||
2020-05-07T04:00:45.608Z,14925983,98,933398.974423113,935446.4151416395,15.991,15.956
|
||||
2020-05-07T04:01:01.565Z,13014161,98,815576.925487247,871912.166688999,15.957,14.926
|
||||
2020-05-07T04:01:16.492Z,16146923,98,1081725.9328733168,938665.4458783861,14.927,17.202
|
||||
2020-05-07T04:01:33.695Z,13374016,98,777423.4726501192,900971.1668014013,17.203,14.844
|
||||
2020-05-07T04:01:48.540Z,15477914,98,1042634.8265409229,938225.9804812997,14.845,16.497
|
||||
2020-05-07T04:02:05.037Z,12597108,98,763599.9272595018,885187.8293865505,16.497,14.231
|
||||
2020-05-07T04:02:19.269Z,13782492,98,968415.6829679596,910697.2380071363,14.232,15.134
|
||||
2020-05-07T04:02:34.405Z,7213889,98,476604.717230444,543337.2749868194,15.136,13.277
|
||||
2020-05-07T04:02:47.682Z,14092413,98,1061415.4552986368,909951.1202944405,13.277,15.487
|
||||
2020-05-07T04:03:03.170Z,14796312,98,955340.3925619835,924422.8414344621,15.488,16.006
|
||||
2020-05-07T04:03:19.177Z,6310417,98,394228.5874929718,522212.59516716324,16.007,12.084
|
||||
2020-05-07T04:03:31.262Z,14718972,98,1217953.8270583367,923862.1641978408,12.085,15.932
|
||||
2020-05-07T04:03:47.195Z,14969131,98,939504.8641184962,911251.6588543252,15.933,16.427
|
||||
2020-05-07T04:04:03.623Z,13218216,98,804615.0474799123,886414.6995708154,16.428,14.912
|
||||
2020-05-07T04:04:18.536Z,14656519,98,982801.5154563133,898566.5501808595,14.913,16.311
|
||||
2020-05-07T04:04:34.847Z,14199628,98,870555.330758384,930940.0118009571,16.311,15.253
|
||||
2020-05-07T04:04:50.100Z,6896315,98,452128.4337507376,529955.8134173519,15.253,13.012999999999998
|
||||
2020-05-07T04:05:03.114Z,13912333,98,1069028.2003995697,914563.0423349987,13.014,15.212
|
||||
2020-05-07T04:05:18.326Z,14618759,98,961001.7749145412,912930.6813214263,15.212,16.013
|
||||
2020-05-07T04:05:34.340Z,15481351,98,966738.5412763832,940886.7752522182,16.014,16.454
|
||||
2020-05-07T04:05:50.795Z,14774041,98,897845.092676998,921247.1783999501,16.455,16.037
|
||||
2020-05-07T04:06:06.833Z,14546853,98,907024.1301907968,921269.9810006333,16.038,15.79
|
||||
2020-05-07T04:06:22.624Z,14626747,98,926271.1037933,925743.4810126582,15.791,15.8
|
||||
2020-05-07T04:06:38.424Z,14275339,98,903502.4683544303,922715.9847456531,15.800000000000002,15.471
|
||||
2020-05-07T04:06:53.895Z,15698454,98,1014701.9585030056,630662.6225293267,15.471,24.892
|
||||
2020-05-07T04:07:18.788Z,13813029,98,554896.1153737998,901987.0053545776,24.893,15.314
|
||||
2020-05-07T04:07:34.102Z,14522315,98,948303.1866266162,895886.1813695249,15.314,16.21
|
||||
2020-05-07T04:07:50.313Z,7565261,98,466674.5419776695,744319.264069264,16.211,10.164
|
||||
2020-05-07T04:08:00.478Z,7667792,98,754332.7102803739,734181.5396399846,10.165,10.444
|
||||
2020-05-07T04:08:10.923Z,12688877,98,1214827.860220201,868506.2970568105,10.445,14.61
|
||||
2020-05-07T04:08:25.534Z,16066350,98,1099606.460885634,930896.923344342,14.611000000000002,17.259
|
||||
2020-05-07T04:08:42.794Z,7019375,98,406684.53070683655,728528.8012454592,17.26,9.635
|
||||
2020-05-07T04:08:52.430Z,14776615,98,1533480.1784973019,914111.660995979,9.636,16.165
|
||||
2020-05-07T04:09:08.595Z,13984887,98,865133.7457469843,880660.3904282115,16.165,15.88
|
||||
2020-05-07T04:09:24.475Z,14696583,98,925477.5188916876,897391.6468217623,15.88,16.377
|
||||
2020-05-07T04:09:40.852Z,14825128,98,905240.7644867803,913721.2942989214,16.377,16.225
|
||||
2020-05-07T04:09:57.077Z,14232487,98,877194.8844375962,891201.4402003756,16.225,15.97
|
||||
2020-05-07T04:10:13.047Z,13776030,98,862619.2861615529,895710.6631989597,15.97,15.38
|
||||
2020-05-07T04:10:28.427Z,14983521,98,974221.1313394018,909856.7524896769,15.38,16.468
|
|
|
@ -0,0 +1,82 @@
|
|||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2009-01.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2009-02.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2009-03.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2009-04.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2009-05.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2009-06.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2009-07.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2009-08.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2009-09.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2009-10.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2009-11.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2009-12.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2010-01.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2010-04.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2010-05.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2010-06.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2010-07.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2010-08.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2010-09.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2010-10.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2010-11.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2010-12.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2011-01.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2011-02.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2011-03.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2011-04.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2011-05.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2011-06.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2011-07.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2011-08.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2011-09.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2011-10.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2011-11.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2011-12.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2012-01.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2012-02.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2012-03.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2012-04.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2012-05.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2012-06.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2012-07.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2012-08.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2012-09.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2012-10.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2012-11.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2012-12.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2013-01.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2013-02.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2013-03.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2013-04.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2013-05.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2013-06.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2013-07.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2013-08.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2013-09.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2013-10.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2013-11.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2013-12.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2014-01.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2014-02.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2014-03.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2014-04.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2014-05.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2014-06.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2014-07.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2014-08.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2014-09.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2014-10.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2014-11.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2014-12.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2019-01.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2019-02.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2019-03.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2019-04.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2019-05.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2019-06.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2019-07.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2019-08.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2019-09.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2019-10.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2019-11.csv
|
||||
https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2019-12.csv
|
|
|
@ -0,0 +1,23 @@
|
|||
# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
|
||||
|
||||
# dependencies
|
||||
/node_modules
|
||||
/.pnp
|
||||
.pnp.js
|
||||
|
||||
# testing
|
||||
/coverage
|
||||
|
||||
# production
|
||||
/build
|
||||
|
||||
# misc
|
||||
.DS_Store
|
||||
.env.local
|
||||
.env.development.local
|
||||
.env.test.local
|
||||
.env.production.local
|
||||
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
|
@ -0,0 +1,119 @@
|
|||
*** This app uses React, React Router and React Responsive. ***
|
||||
|
||||
*** To change settings in this app, in App.js ***
|
||||
|
||||
1. Set themes and query descriptions in the state.
|
||||
2. Create routes to display pairs of meters (CPU, FPGA) in either Grid (row, col) or Tabs components:
|
||||
a. Pass selected record data sets, styled themes and query descriptions.
|
||||
b. Pass scalar to set display size (defaults to 1)
|
||||
c. Pass effects to turn on/off blur and saturation filters on the Meter component (defaults to true).
|
||||
d. Use React Responsive Media Query with scalar for responsive scaling of the pairs of meters.
|
||||
|
||||
*** The Project Hierarchy (with descriptions) ***
|
||||
|
||||
public:
|
||||
themes: (each folder contains color themed jpg and svg images for meter component)
|
||||
blue:
|
||||
green:
|
||||
green2:
|
||||
magenta:
|
||||
orange:
|
||||
purple:
|
||||
src:
|
||||
components:
|
||||
App.js - loads the data, sets up the themes and query descriptions and handles the routing
|
||||
Grid.jsx - composes the pairs of meters in a centered row of columns
|
||||
Query.jsx - composes the paired CPU and FPGA components in a centered row of columns
|
||||
CPU.jsx - presents a meter using CPU specific settings
|
||||
FPGA.jsx - presents a meter using FPGA specific settings
|
||||
Meter.jsx - displays a simulation of a live metering of data processing
|
||||
ProgressGrower.jsx - displays a custom made progress bar
|
||||
unused:
|
||||
ProgressSlider.jsx - displays a custom progress spinner
|
||||
Tabs.jsx - composes the pairs of meters in a tab navigation
|
||||
data:
|
||||
cpu.json - stores the data in the cpu csv files as arrays of records
|
||||
fpga.json - stores the data in the fpgs csv files as arrays of records
|
||||
|
||||
models:
|
||||
Record.js - a javascript class model for a row of data from the CPU and FPGA csv query files
|
||||
Theme.js - a javascript class model for a styled theme for the Grid and Meter components
|
||||
queries:
|
||||
cpu:
|
||||
fpga:
|
||||
utilities:
|
||||
count.js - a node js app to verify the total data processed
|
||||
csv2json.js - a node js app to convert csv data files to json (reads all files in src/queries folder and writes json files to src/data folder).
|
||||
|
||||
|
||||
|
||||
*** THE FOLLOWING IS THE AUTO GENERATED README FOR PROJECTS BOOTSTRAPPED USING CREATE REACT APP ***
|
||||
|
||||
This project was bootstrapped with [Create React App](https://github.com/facebook/create-react-app).
|
||||
|
||||
## Available Scripts
|
||||
|
||||
In the project directory, you can run:
|
||||
|
||||
### `npm start`
|
||||
|
||||
Runs the app in the development mode.<br />
|
||||
Open [http://localhost:3000](http://localhost:3000) to view it in the browser.
|
||||
|
||||
The page will reload if you make edits.<br />
|
||||
You will also see any lint errors in the console.
|
||||
|
||||
### `npm test`
|
||||
|
||||
Launches the test runner in the interactive watch mode.<br />
|
||||
See the section about [running tests](https://facebook.github.io/create-react-app/docs/running-tests) for more information.
|
||||
|
||||
### `npm run build`
|
||||
|
||||
Builds the app for production to the `build` folder.<br />
|
||||
It correctly bundles React in production mode and optimizes the build for the best performance.
|
||||
|
||||
The build is minified and the filenames include the hashes.<br />
|
||||
Your app is ready to be deployed!
|
||||
|
||||
See the section about [deployment](https://facebook.github.io/create-react-app/docs/deployment) for more information.
|
||||
|
||||
### `npm run eject`
|
||||
|
||||
**Note: this is a one-way operation. Once you `eject`, you can’t go back!**
|
||||
|
||||
If you aren’t satisfied with the build tool and configuration choices, you can `eject` at any time. This command will remove the single build dependency from your project.
|
||||
|
||||
Instead, it will copy all the configuration files and the transitive dependencies (webpack, Babel, ESLint, etc) right into your project so you have full control over them. All of the commands except `eject` will still work, but they will point to the copied scripts so you can tweak them. At this point you’re on your own.
|
||||
|
||||
You don’t have to ever use `eject`. The curated feature set is suitable for small and middle deployments, and you shouldn’t feel obligated to use this feature. However we understand that this tool wouldn’t be useful if you couldn’t customize it when you are ready for it.
|
||||
|
||||
## Learn More
|
||||
|
||||
You can learn more in the [Create React App documentation](https://facebook.github.io/create-react-app/docs/getting-started).
|
||||
|
||||
To learn React, check out the [React documentation](https://reactjs.org/).
|
||||
|
||||
### Code Splitting
|
||||
|
||||
This section has moved here: https://facebook.github.io/create-react-app/docs/code-splitting
|
||||
|
||||
### Analyzing the Bundle Size
|
||||
|
||||
This section has moved here: https://facebook.github.io/create-react-app/docs/analyzing-the-bundle-size
|
||||
|
||||
### Making a Progressive Web App
|
||||
|
||||
This section has moved here: https://facebook.github.io/create-react-app/docs/making-a-progressive-web-app
|
||||
|
||||
### Advanced Configuration
|
||||
|
||||
This section has moved here: https://facebook.github.io/create-react-app/docs/advanced-configuration
|
||||
|
||||
### Deployment
|
||||
|
||||
This section has moved here: https://facebook.github.io/create-react-app/docs/deployment
|
||||
|
||||
### `npm run build` fails to minify
|
||||
|
||||
This section has moved here: https://facebook.github.io/create-react-app/docs/troubleshooting#npm-run-build-fails-to-minify
|
|
@ -0,0 +1,38 @@
|
|||
{
|
||||
"name": "nealanalyticsdemo",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"dependencies": {
|
||||
"@testing-library/jest-dom": "^4.2.4",
|
||||
"@testing-library/react": "^9.5.0",
|
||||
"@testing-library/user-event": "^7.2.1",
|
||||
"axios": "^0.19.2",
|
||||
"curry": "^1.2.0",
|
||||
"react": "^16.13.1",
|
||||
"react-dom": "^16.13.1",
|
||||
"react-responsive": "^8.0.3",
|
||||
"react-router-dom": "^5.1.2",
|
||||
"react-scripts": "3.4.1"
|
||||
},
|
||||
"scripts": {
|
||||
"start": "react-scripts start",
|
||||
"build": "react-scripts build",
|
||||
"test": "react-scripts test",
|
||||
"eject": "react-scripts eject"
|
||||
},
|
||||
"eslintConfig": {
|
||||
"extends": "react-app"
|
||||
},
|
||||
"browserslist": {
|
||||
"production": [
|
||||
">0.2%",
|
||||
"not dead",
|
||||
"not op_mini all"
|
||||
],
|
||||
"development": [
|
||||
"last 1 chrome version",
|
||||
"last 1 firefox version",
|
||||
"last 1 safari version"
|
||||
]
|
||||
}
|
||||
}
|
После Ширина: | Высота: | Размер: 3.1 KiB |
|
@ -0,0 +1,43 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<link rel="icon" href="%PUBLIC_URL%/favicon.ico" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<meta name="theme-color" content="#000000" />
|
||||
<meta
|
||||
name="description"
|
||||
content="Web site created using create-react-app"
|
||||
/>
|
||||
<link rel="apple-touch-icon" href="%PUBLIC_URL%/logo192.png" />
|
||||
<!--
|
||||
manifest.json provides metadata used when your web app is installed on a
|
||||
user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
|
||||
-->
|
||||
<link rel="manifest" href="%PUBLIC_URL%/manifest.json" />
|
||||
<!--
|
||||
Notice the use of %PUBLIC_URL% in the tags above.
|
||||
It will be replaced with the URL of the `public` folder during the build.
|
||||
Only files inside the `public` folder can be referenced from the HTML.
|
||||
|
||||
Unlike "/favicon.ico" or "favicon.ico", "%PUBLIC_URL%/favicon.ico" will
|
||||
work correctly both with client-side routing and a non-root public URL.
|
||||
Learn how to configure a non-root public URL by running `npm run build`.
|
||||
-->
|
||||
<title>React App</title>
|
||||
</head>
|
||||
<body style="background: url(themes/orange/background.jpg) no-repeat center center fixed;-webkit-background-size: cover;-moz-background-size: cover;-o-background-size: cover;background-size: cover;">
|
||||
<noscript>You need to enable JavaScript to run this app.</noscript>
|
||||
<div id="root"></div>
|
||||
<!--
|
||||
This HTML file is a template.
|
||||
If you open it directly in the browser, you will see an empty page.
|
||||
|
||||
You can add webfonts, meta tags, or analytics to this file.
|
||||
The build step will place the bundled scripts into the <body> tag.
|
||||
|
||||
To begin the development, run `npm start` or `yarn start`.
|
||||
To create a production bundle, use `npm run build` or `yarn build`.
|
||||
-->
|
||||
</body>
|
||||
</html>
|
После Ширина: | Высота: | Размер: 5.2 KiB |
После Ширина: | Высота: | Размер: 9.4 KiB |
|
@ -0,0 +1,25 @@
|
|||
{
|
||||
"short_name": "React App",
|
||||
"name": "Create React App Sample",
|
||||
"icons": [
|
||||
{
|
||||
"src": "favicon.ico",
|
||||
"sizes": "64x64 32x32 24x24 16x16",
|
||||
"type": "image/x-icon"
|
||||
},
|
||||
{
|
||||
"src": "logo192.png",
|
||||
"type": "image/png",
|
||||
"sizes": "192x192"
|
||||
},
|
||||
{
|
||||
"src": "logo512.png",
|
||||
"type": "image/png",
|
||||
"sizes": "512x512"
|
||||
}
|
||||
],
|
||||
"start_url": ".",
|
||||
"display": "standalone",
|
||||
"theme_color": "#000000",
|
||||
"background_color": "#ffffff"
|
||||
}
|
|
@ -0,0 +1,3 @@
|
|||
# https://www.robotstxt.org/robotstxt.html
|
||||
User-agent: *
|
||||
Disallow:
|
После Ширина: | Высота: | Размер: 194 KiB |
После Ширина: | Высота: | Размер: 39 KiB |
После Ширина: | Высота: | Размер: 43 KiB |
|
@ -0,0 +1 @@
|
|||
<svg width="18" height="62" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" overflow="hidden"><defs><clipPath id="clip0"><path d="M112 222 130 222 130 284 112 284Z" fill-rule="evenodd" clip-rule="evenodd"/></clipPath></defs><g clip-path="url(#clip0)" transform="translate(-112 -222)"><path d="M124.478 267.398 120.375 222.774 117.887 267.269C113.798 269.031 111.911 273.773 113.672 277.861 115.434 281.95 120.176 283.837 124.264 282.076 128.353 280.314 130.24 275.573 128.479 271.484 127.7 269.675 126.286 268.215 124.504 267.377Z" fill="#FFFFFF" fill-rule="evenodd"/></g></svg>
|
После Ширина: | Высота: | Размер: 608 B |
|
@ -0,0 +1 @@
|
|||
<svg width="136" height="52" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" overflow="hidden"><defs><clipPath id="clip0"><path d="M66 521 202 521 202 573 66 573Z" fill-rule="evenodd" clip-rule="evenodd"/></clipPath></defs><g clip-path="url(#clip0)" transform="translate(-66 -521)"><path d="M67 543.5C67 533.835 74.835 526 84.5 526L183.5 526C193.165 526 201 533.835 201 543.5L201 543.5C201 553.165 193.165 561 183.5 561L84.5 561C74.835 561 67 553.165 67 543.5Z" fill="#0882D8" fill-rule="evenodd"/><text fill="#FFFFFF" font-family="Segoe UI,Segoe UI_MSFontService,sans-serif" font-weight="400" font-size="24" transform="translate(101.401 552)">START</text></g></svg>
|
После Ширина: | Высота: | Размер: 696 B |
|
@ -0,0 +1 @@
|
|||
<svg width="136" height="52" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" overflow="hidden"><defs><clipPath id="clip0"><path d="M1111 521 1247 521 1247 573 1111 573Z" fill-rule="evenodd" clip-rule="evenodd"/></clipPath></defs><g clip-path="url(#clip0)" transform="translate(-1111 -521)"><path d="M1112 543.5C1112 533.835 1119.83 526 1129.5 526L1228.5 526C1238.17 526 1246 533.835 1246 543.5L1246 543.5C1246 553.165 1238.16 561 1228.5 561L1129.5 561C1119.83 561 1112 553.165 1112 543.5Z" fill="#8CDDF7" fill-rule="evenodd"/><text font-family="Segoe UI,Segoe UI_MSFontService,sans-serif" font-weight="400" font-size="24" transform="translate(1146.16 552)">START</text></g></svg>
|
После Ширина: | Высота: | Размер: 709 B |
|
@ -0,0 +1 @@
|
|||
<svg width="136" height="52" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" overflow="hidden"><defs><clipPath id="clip0"><path d="M66 567 202 567 202 619 66 619Z" fill-rule="evenodd" clip-rule="evenodd"/></clipPath></defs><g clip-path="url(#clip0)" transform="translate(-66 -567)"><path d="M67 589.5C67 579.835 74.835 572 84.5 572L183.5 572C193.165 572 201 579.835 201 589.5L201 589.5C201 599.165 193.165 607 183.5 607L84.5 607C74.835 607 67 599.165 67 589.5Z" fill="#0882D8" fill-rule="evenodd"/><text fill="#FFFFFF" font-family="Segoe UI,Segoe UI_MSFontService,sans-serif" font-weight="400" font-size="24" transform="translate(106.061 598)">STOP</text></g></svg>
|
После Ширина: | Высота: | Размер: 695 B |
|
@ -0,0 +1 @@
|
|||
<svg width="136" height="52" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" overflow="hidden"><defs><clipPath id="clip0"><path d="M1111 567 1247 567 1247 619 1111 619Z" fill-rule="evenodd" clip-rule="evenodd"/></clipPath></defs><g clip-path="url(#clip0)" transform="translate(-1111 -567)"><path d="M1112 589.5C1112 579.835 1119.83 572 1129.5 572L1228.5 572C1238.17 572 1246 579.835 1246 589.5L1246 589.5C1246 599.165 1238.16 607 1228.5 607L1129.5 607C1119.83 607 1112 599.165 1112 589.5Z" fill="#8CDDF7" fill-rule="evenodd"/><text font-family="Segoe UI,Segoe UI_MSFontService,sans-serif" font-weight="400" font-size="24" transform="translate(1150.82 598)">STOP</text></g></svg>
|
После Ширина: | Высота: | Размер: 708 B |
После Ширина: | Высота: | Размер: 194 KiB |
После Ширина: | Высота: | Размер: 40 KiB |
После Ширина: | Высота: | Размер: 42 KiB |
|
@ -0,0 +1 @@
|
|||
<svg width="18" height="62" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" overflow="hidden"><defs><clipPath id="clip0"><path d="M112 222 130 222 130 284 112 284Z" fill-rule="evenodd" clip-rule="evenodd"/></clipPath></defs><g clip-path="url(#clip0)" transform="translate(-112 -222)"><path d="M124.478 267.398 120.375 222.774 117.887 267.269C113.798 269.031 111.911 273.773 113.672 277.861 115.434 281.95 120.176 283.837 124.264 282.076 128.353 280.314 130.24 275.573 128.479 271.484 127.7 269.675 126.286 268.215 124.504 267.377Z" fill="#FFFFFF" fill-rule="evenodd"/></g></svg>
|
После Ширина: | Высота: | Размер: 608 B |
|
@ -0,0 +1 @@
|
|||
<svg width="136" height="52" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" overflow="hidden"><defs><clipPath id="clip0"><path d="M66 521 202 521 202 573 66 573Z" fill-rule="evenodd" clip-rule="evenodd"/></clipPath></defs><g clip-path="url(#clip0)" transform="translate(-66 -521)"><path d="M67 543.5C67 533.835 74.835 526 84.5 526L183.5 526C193.165 526 201 533.835 201 543.5L201 543.5C201 553.165 193.165 561 183.5 561L84.5 561C74.835 561 67 553.165 67 543.5Z" fill="#44683A" fill-rule="evenodd"/><path d="M67 543.5C67 533.835 74.835 526 84.5 526L183.5 526C193.165 526 201 533.835 201 543.5L201 543.5C201 553.165 193.165 561 183.5 561L84.5 561C74.835 561 67 553.165 67 543.5Z" fill="#44683A" fill-rule="evenodd"/><text fill="#FFFFFF" font-family="Segoe UI,Segoe UI_MSFontService,sans-serif" font-weight="400" font-size="24" transform="translate(101.401 552)">START</text></g></svg>
|
После Ширина: | Высота: | Размер: 912 B |
|
@ -0,0 +1 @@
|
|||
<svg width="136" height="52" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" overflow="hidden"><defs><clipPath id="clip0"><path d="M1111 521 1247 521 1247 573 1111 573Z" fill-rule="evenodd" clip-rule="evenodd"/></clipPath></defs><g clip-path="url(#clip0)" transform="translate(-1111 -521)"><path d="M1112 543.5C1112 533.835 1119.83 526 1129.5 526L1228.5 526C1238.17 526 1246 533.835 1246 543.5L1246 543.5C1246 553.165 1238.16 561 1228.5 561L1129.5 561C1119.83 561 1112 553.165 1112 543.5Z" fill="#82B74F" fill-rule="evenodd"/><text font-family="Segoe UI,Segoe UI_MSFontService,sans-serif" font-weight="400" font-size="24" transform="translate(1146.16 552)">START</text></g></svg>
|
После Ширина: | Высота: | Размер: 709 B |
|
@ -0,0 +1 @@
|
|||
<svg width="136" height="52" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" overflow="hidden"><defs><clipPath id="clip0"><path d="M66 567 202 567 202 619 66 619Z" fill-rule="evenodd" clip-rule="evenodd"/></clipPath></defs><g clip-path="url(#clip0)" transform="translate(-66 -567)"><path d="M67 589.5C67 579.835 74.835 572 84.5 572L183.5 572C193.165 572 201 579.835 201 589.5L201 589.5C201 599.165 193.165 607 183.5 607L84.5 607C74.835 607 67 599.165 67 589.5Z" fill="#44683A" fill-rule="evenodd"/><path d="M67 589.5C67 579.835 74.835 572 84.5 572L183.5 572C193.165 572 201 579.835 201 589.5L201 589.5C201 599.165 193.165 607 183.5 607L84.5 607C74.835 607 67 599.165 67 589.5Z" fill="#44683A" fill-rule="evenodd"/><text fill="#FFFFFF" font-family="Segoe UI,Segoe UI_MSFontService,sans-serif" font-weight="400" font-size="24" transform="translate(106.061 598)">STOP</text></g></svg>
|
После Ширина: | Высота: | Размер: 911 B |
|
@ -0,0 +1 @@
|
|||
<svg width="136" height="52" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" overflow="hidden"><defs><clipPath id="clip0"><path d="M1111 567 1247 567 1247 619 1111 619Z" fill-rule="evenodd" clip-rule="evenodd"/></clipPath></defs><g clip-path="url(#clip0)" transform="translate(-1111 -567)"><path d="M1112 589.5C1112 579.835 1119.83 572 1129.5 572L1228.5 572C1238.17 572 1246 579.835 1246 589.5L1246 589.5C1246 599.165 1238.16 607 1228.5 607L1129.5 607C1119.83 607 1112 599.165 1112 589.5Z" fill="#82B74F" fill-rule="evenodd"/><text font-family="Segoe UI,Segoe UI_MSFontService,sans-serif" font-weight="400" font-size="24" transform="translate(1150.82 598)">STOP</text></g></svg>
|
После Ширина: | Высота: | Размер: 708 B |
После Ширина: | Высота: | Размер: 194 KiB |
После Ширина: | Высота: | Размер: 40 KiB |
После Ширина: | Высота: | Размер: 42 KiB |
|
@ -0,0 +1 @@
|
|||
<svg width="18" height="62" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" overflow="hidden"><defs><clipPath id="clip0"><path d="M112 222 130 222 130 284 112 284Z" fill-rule="evenodd" clip-rule="evenodd"/></clipPath></defs><g clip-path="url(#clip0)" transform="translate(-112 -222)"><path d="M124.478 267.398 120.375 222.774 117.887 267.269C113.798 269.031 111.911 273.773 113.672 277.861 115.434 281.95 120.176 283.837 124.264 282.076 128.353 280.314 130.24 275.573 128.479 271.484 127.7 269.675 126.286 268.215 124.504 267.377Z" fill="#FFFFFF" fill-rule="evenodd"/></g></svg>
|
После Ширина: | Высота: | Размер: 608 B |
|
@ -0,0 +1 @@
|
|||
<svg width="136" height="52" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" overflow="hidden"><defs><clipPath id="clip0"><path d="M66 521 202 521 202 573 66 573Z" fill-rule="evenodd" clip-rule="evenodd"/></clipPath></defs><g clip-path="url(#clip0)" transform="translate(-66 -521)"><path d="M67 543.5C67 533.835 74.835 526 84.5 526L183.5 526C193.165 526 201 533.835 201 543.5L201 543.5C201 553.165 193.165 561 183.5 561L84.5 561C74.835 561 67 553.165 67 543.5Z" fill="#308950" fill-rule="evenodd"/><text fill="#FFFFFF" font-family="Segoe UI,Segoe UI_MSFontService,sans-serif" font-weight="400" font-size="24" transform="translate(101.401 552)">START</text></g></svg>
|
После Ширина: | Высота: | Размер: 696 B |
|
@ -0,0 +1 @@
|
|||
<svg width="136" height="52" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" overflow="hidden"><defs><clipPath id="clip0"><path d="M1111 521 1247 521 1247 573 1111 573Z" fill-rule="evenodd" clip-rule="evenodd"/></clipPath></defs><g clip-path="url(#clip0)" transform="translate(-1111 -521)"><path d="M1112 543.5C1112 533.835 1119.83 526 1129.5 526L1228.5 526C1238.17 526 1246 533.835 1246 543.5L1246 543.5C1246 553.165 1238.16 561 1228.5 561L1129.5 561C1119.83 561 1112 553.165 1112 543.5Z" fill="#80B173" fill-rule="evenodd"/><text font-family="Segoe UI,Segoe UI_MSFontService,sans-serif" font-weight="400" font-size="24" transform="translate(1146.16 552)">START</text></g></svg>
|
После Ширина: | Высота: | Размер: 709 B |
|
@ -0,0 +1 @@
|
|||
<svg width="136" height="52" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" overflow="hidden"><defs><clipPath id="clip0"><path d="M66 567 202 567 202 619 66 619Z" fill-rule="evenodd" clip-rule="evenodd"/></clipPath></defs><g clip-path="url(#clip0)" transform="translate(-66 -567)"><path d="M67 589.5C67 579.835 74.835 572 84.5 572L183.5 572C193.165 572 201 579.835 201 589.5L201 589.5C201 599.165 193.165 607 183.5 607L84.5 607C74.835 607 67 599.165 67 589.5Z" fill="#308950" fill-rule="evenodd"/><text fill="#FFFFFF" font-family="Segoe UI,Segoe UI_MSFontService,sans-serif" font-weight="400" font-size="24" transform="translate(106.061 598)">STOP</text></g></svg>
|
После Ширина: | Высота: | Размер: 695 B |
|
@ -0,0 +1 @@
|
|||
<svg width="136" height="52" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" overflow="hidden"><defs><clipPath id="clip0"><path d="M1111 567 1247 567 1247 619 1111 619Z" fill-rule="evenodd" clip-rule="evenodd"/></clipPath></defs><g clip-path="url(#clip0)" transform="translate(-1111 -567)"><path d="M1112 589.5C1112 579.835 1119.83 572 1129.5 572L1228.5 572C1238.17 572 1246 579.835 1246 589.5L1246 589.5C1246 599.165 1238.16 607 1228.5 607L1129.5 607C1119.83 607 1112 599.165 1112 589.5Z" fill="#80B173" fill-rule="evenodd"/><text font-family="Segoe UI,Segoe UI_MSFontService,sans-serif" font-weight="400" font-size="24" transform="translate(1150.82 598)">STOP</text></g></svg>
|
После Ширина: | Высота: | Размер: 708 B |
После Ширина: | Высота: | Размер: 194 KiB |
После Ширина: | Высота: | Размер: 40 KiB |
После Ширина: | Высота: | Размер: 42 KiB |
|
@ -0,0 +1 @@
|
|||
<svg width="18" height="62" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" overflow="hidden"><defs><clipPath id="clip0"><path d="M112 222 130 222 130 284 112 284Z" fill-rule="evenodd" clip-rule="evenodd"/></clipPath></defs><g clip-path="url(#clip0)" transform="translate(-112 -222)"><path d="M124.478 267.398 120.375 222.774 117.887 267.269C113.798 269.031 111.911 273.773 113.672 277.861 115.434 281.95 120.176 283.837 124.264 282.076 128.353 280.314 130.24 275.573 128.479 271.484 127.7 269.675 126.286 268.215 124.504 267.377Z" fill="#FFFFFF" fill-rule="evenodd"/></g></svg>
|
После Ширина: | Высота: | Размер: 608 B |
|
@ -0,0 +1 @@
|
|||
<svg width="136" height="52" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" overflow="hidden"><defs><clipPath id="clip0"><path d="M66 521 202 521 202 573 66 573Z" fill-rule="evenodd" clip-rule="evenodd"/></clipPath></defs><g clip-path="url(#clip0)" transform="translate(-66 -521)"><path d="M67 543.5C67 533.835 74.835 526 84.5 526L183.5 526C193.165 526 201 533.835 201 543.5L201 543.5C201 553.165 193.165 561 183.5 561L84.5 561C74.835 561 67 553.165 67 543.5Z" fill="#B4009E" fill-rule="evenodd"/><text fill="#FFFFFF" font-family="Segoe UI,Segoe UI_MSFontService,sans-serif" font-weight="400" font-size="24" transform="translate(101.401 552)">START</text></g></svg>
|
После Ширина: | Высота: | Размер: 696 B |
|
@ -0,0 +1 @@
|
|||
<svg width="136" height="52" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" overflow="hidden"><defs><clipPath id="clip0"><path d="M1111 521 1247 521 1247 573 1111 573Z" fill-rule="evenodd" clip-rule="evenodd"/></clipPath></defs><g clip-path="url(#clip0)" transform="translate(-1111 -521)"><path d="M1112 543.5C1112 533.835 1119.83 526 1129.5 526L1228.5 526C1238.17 526 1246 533.835 1246 543.5L1246 543.5C1246 553.165 1238.16 561 1228.5 561L1129.5 561C1119.83 561 1112 553.165 1112 543.5Z" fill="#E3008C" fill-rule="evenodd"/><text fill="#FFFFFF" font-family="Segoe UI,Segoe UI_MSFontService,sans-serif" font-weight="400" font-size="24" transform="translate(1146.16 552)">START</text></g></svg>
|
После Ширина: | Высота: | Размер: 724 B |
|
@ -0,0 +1 @@
|
|||
<svg width="136" height="52" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" overflow="hidden"><defs><clipPath id="clip0"><path d="M66 567 202 567 202 619 66 619Z" fill-rule="evenodd" clip-rule="evenodd"/></clipPath></defs><g clip-path="url(#clip0)" transform="translate(-66 -567)"><path d="M67 589.5C67 579.835 74.835 572 84.5 572L183.5 572C193.165 572 201 579.835 201 589.5L201 589.5C201 599.165 193.165 607 183.5 607L84.5 607C74.835 607 67 599.165 67 589.5Z" fill="#B4009E" fill-rule="evenodd"/><text fill="#FFFFFF" font-family="Segoe UI,Segoe UI_MSFontService,sans-serif" font-weight="400" font-size="24" transform="translate(106.061 598)">STOP</text></g></svg>
|
После Ширина: | Высота: | Размер: 695 B |
|
@ -0,0 +1 @@
|
|||
<svg width="136" height="52" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" overflow="hidden"><defs><clipPath id="clip0"><path d="M1111 567 1247 567 1247 619 1111 619Z" fill-rule="evenodd" clip-rule="evenodd"/></clipPath></defs><g clip-path="url(#clip0)" transform="translate(-1111 -567)"><path d="M1112 589.5C1112 579.835 1119.83 572 1129.5 572L1228.5 572C1238.17 572 1246 579.835 1246 589.5L1246 589.5C1246 599.165 1238.16 607 1228.5 607L1129.5 607C1119.83 607 1112 599.165 1112 589.5Z" fill="#E3008C" fill-rule="evenodd"/><text fill="#FFFFFF" font-family="Segoe UI,Segoe UI_MSFontService,sans-serif" font-weight="400" font-size="24" transform="translate(1150.82 598)">STOP</text></g></svg>
|
После Ширина: | Высота: | Размер: 723 B |
После Ширина: | Высота: | Размер: 194 KiB |
После Ширина: | Высота: | Размер: 34 KiB |
После Ширина: | Высота: | Размер: 42 KiB |
|
@ -0,0 +1 @@
|
|||
<svg width="18" height="62" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" overflow="hidden"><defs><clipPath id="clip0"><path d="M112 222 130 222 130 284 112 284Z" fill-rule="evenodd" clip-rule="evenodd"/></clipPath></defs><g clip-path="url(#clip0)" transform="translate(-112 -222)"><path d="M124.478 267.398 120.375 222.774 117.887 267.269C113.798 269.031 111.911 273.773 113.672 277.861 115.434 281.95 120.176 283.837 124.264 282.076 128.353 280.314 130.24 275.573 128.479 271.484 127.7 269.675 126.286 268.215 124.504 267.377Z" fill="#FFFFFF" fill-rule="evenodd"/></g></svg>
|
После Ширина: | Высота: | Размер: 608 B |
|
@ -0,0 +1 @@
|
|||
<svg width="136" height="52" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" overflow="hidden"><defs><clipPath id="clip0"><path d="M66 521 202 521 202 573 66 573Z" fill-rule="evenodd" clip-rule="evenodd"/></clipPath></defs><g clip-path="url(#clip0)" transform="translate(-66 -521)"><path d="M67 543.5C67 533.835 74.835 526 84.5 526L183.5 526C193.165 526 201 533.835 201 543.5L201 543.5C201 553.165 193.165 561 183.5 561L84.5 561C74.835 561 67 553.165 67 543.5Z" fill="#E16A1A" fill-rule="evenodd"/><text fill="#FFFFFF" font-family="Segoe UI,Segoe UI_MSFontService,sans-serif" font-weight="400" font-size="24" transform="translate(101.401 552)">START</text></g></svg>
|
После Ширина: | Высота: | Размер: 696 B |
|
@ -0,0 +1 @@
|
|||
<svg width="136" height="52" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" overflow="hidden"><defs><clipPath id="clip0"><path d="M1111 521 1247 521 1247 573 1111 573Z" fill-rule="evenodd" clip-rule="evenodd"/></clipPath></defs><g clip-path="url(#clip0)" transform="translate(-1111 -521)"><path d="M1112 543.5C1112 533.835 1119.83 526 1129.5 526L1228.5 526C1238.17 526 1246 533.835 1246 543.5L1246 543.5C1246 553.165 1238.16 561 1228.5 561L1129.5 561C1119.83 561 1112 553.165 1112 543.5Z" fill="#F3AA6D" fill-rule="evenodd"/><text font-family="Segoe UI,Segoe UI_MSFontService,sans-serif" font-weight="400" font-size="24" transform="translate(1146.16 552)">START</text></g></svg>
|
После Ширина: | Высота: | Размер: 709 B |
|
@ -0,0 +1 @@
|
|||
<svg width="136" height="52" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" overflow="hidden"><defs><clipPath id="clip0"><path d="M66 567 202 567 202 619 66 619Z" fill-rule="evenodd" clip-rule="evenodd"/></clipPath></defs><g clip-path="url(#clip0)" transform="translate(-66 -567)"><path d="M67 589.5C67 579.835 74.835 572 84.5 572L183.5 572C193.165 572 201 579.835 201 589.5L201 589.5C201 599.165 193.165 607 183.5 607L84.5 607C74.835 607 67 599.165 67 589.5Z" fill="#E16A1A" fill-rule="evenodd"/><text fill="#FFFFFF" font-family="Segoe UI,Segoe UI_MSFontService,sans-serif" font-weight="400" font-size="24" transform="translate(106.061 598)">STOP</text></g></svg>
|
После Ширина: | Высота: | Размер: 695 B |
|
@ -0,0 +1 @@
|
|||
<svg width="136" height="52" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" overflow="hidden"><defs><clipPath id="clip0"><path d="M1111 567 1247 567 1247 619 1111 619Z" fill-rule="evenodd" clip-rule="evenodd"/></clipPath></defs><g clip-path="url(#clip0)" transform="translate(-1111 -567)"><path d="M1112 589.5C1112 579.835 1119.83 572 1129.5 572L1228.5 572C1238.17 572 1246 579.835 1246 589.5L1246 589.5C1246 599.165 1238.16 607 1228.5 607L1129.5 607C1119.83 607 1112 599.165 1112 589.5Z" fill="#F3AA6D" fill-rule="evenodd"/><text font-family="Segoe UI,Segoe UI_MSFontService,sans-serif" font-weight="400" font-size="24" transform="translate(1150.82 598)">STOP</text></g></svg>
|
После Ширина: | Высота: | Размер: 708 B |
После Ширина: | Высота: | Размер: 194 KiB |
После Ширина: | Высота: | Размер: 40 KiB |
После Ширина: | Высота: | Размер: 42 KiB |
|
@ -0,0 +1 @@
|
|||
<svg width="18" height="62" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" overflow="hidden"><defs><clipPath id="clip0"><path d="M112 222 130 222 130 284 112 284Z" fill-rule="evenodd" clip-rule="evenodd"/></clipPath></defs><g clip-path="url(#clip0)" transform="translate(-112 -222)"><path d="M124.478 267.398 120.375 222.774 117.887 267.269C113.798 269.031 111.911 273.773 113.672 277.861 115.434 281.95 120.176 283.837 124.264 282.076 128.353 280.314 130.24 275.573 128.479 271.484 127.7 269.675 126.286 268.215 124.504 267.377Z" fill="#FFFFFF" fill-rule="evenodd"/></g></svg>
|
После Ширина: | Высота: | Размер: 608 B |
|
@ -0,0 +1 @@
|
|||
<svg width="136" height="52" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" overflow="hidden"><defs><clipPath id="clip0"><path d="M66 521 202 521 202 573 66 573Z" fill-rule="evenodd" clip-rule="evenodd"/></clipPath></defs><g clip-path="url(#clip0)" transform="translate(-66 -521)"><path d="M67 543.5C67 533.835 74.835 526 84.5 526L183.5 526C193.165 526 201 533.835 201 543.5L201 543.5C201 553.165 193.165 561 183.5 561L84.5 561C74.835 561 67 553.165 67 543.5Z" fill="#5059C8" fill-rule="evenodd"/><text fill="#FFFFFF" font-family="Segoe UI,Segoe UI_MSFontService,sans-serif" font-weight="400" font-size="24" transform="translate(101.401 552)">START</text></g></svg>
|
После Ширина: | Высота: | Размер: 696 B |
|
@ -0,0 +1 @@
|
|||
<svg width="136" height="52" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" overflow="hidden"><defs><clipPath id="clip0"><path d="M1111 521 1247 521 1247 573 1111 573Z" fill-rule="evenodd" clip-rule="evenodd"/></clipPath></defs><g clip-path="url(#clip0)" transform="translate(-1111 -521)"><path d="M1112 543.5C1112 533.835 1119.83 526 1129.5 526L1228.5 526C1238.17 526 1246 533.835 1246 543.5L1246 543.5C1246 553.165 1238.16 561 1228.5 561L1129.5 561C1119.83 561 1112 553.165 1112 543.5Z" fill="#7B83EB" fill-rule="evenodd"/><text fill="#FFFFFF" font-family="Segoe UI,Segoe UI_MSFontService,sans-serif" font-weight="400" font-size="24" transform="translate(1146.16 552)">START</text></g></svg>
|
После Ширина: | Высота: | Размер: 724 B |
|
@ -0,0 +1 @@
|
|||
<svg width="136" height="52" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" overflow="hidden"><defs><clipPath id="clip0"><path d="M66 567 202 567 202 619 66 619Z" fill-rule="evenodd" clip-rule="evenodd"/></clipPath></defs><g clip-path="url(#clip0)" transform="translate(-66 -567)"><path d="M67 589.5C67 579.835 74.835 572 84.5 572L183.5 572C193.165 572 201 579.835 201 589.5L201 589.5C201 599.165 193.165 607 183.5 607L84.5 607C74.835 607 67 599.165 67 589.5Z" fill="#5059C8" fill-rule="evenodd"/><text fill="#FFFFFF" font-family="Segoe UI,Segoe UI_MSFontService,sans-serif" font-weight="400" font-size="24" transform="translate(106.061 598)">STOP</text></g></svg>
|
После Ширина: | Высота: | Размер: 695 B |
|
@ -0,0 +1 @@
|
|||
<svg width="136" height="52" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" overflow="hidden"><defs><clipPath id="clip0"><path d="M1111 567 1247 567 1247 619 1111 619Z" fill-rule="evenodd" clip-rule="evenodd"/></clipPath></defs><g clip-path="url(#clip0)" transform="translate(-1111 -567)"><path d="M1112 589.5C1112 579.835 1119.83 572 1129.5 572L1228.5 572C1238.17 572 1246 579.835 1246 589.5L1246 589.5C1246 599.165 1238.16 607 1228.5 607L1129.5 607C1119.83 607 1112 599.165 1112 589.5Z" fill="#7B83EB" fill-rule="evenodd"/><text fill="#FFFFFF" font-family="Segoe UI,Segoe UI_MSFontService,sans-serif" font-weight="400" font-size="24" transform="translate(1150.82 598)">STOP</text></g></svg>
|
После Ширина: | Высота: | Размер: 723 B |
|
@ -0,0 +1,7 @@
|
|||
ul {
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
li {
|
||||
list-style-type: none;
|
||||
}
|
|
@ -0,0 +1,172 @@
|
|||
import React from 'react';
|
||||
import './App.css';
|
||||
import Record from './models/Record';
|
||||
import Theme from './models/Theme';
|
||||
import cpu from './data/cpu.json';
|
||||
import fpga from './data/fpga.json';
|
||||
import Grid from './components/Grid';
|
||||
import {
|
||||
BrowserRouter as Router,
|
||||
Switch,
|
||||
Route
|
||||
} from 'react-router-dom';
|
||||
|
||||
class App extends React.Component {
|
||||
constructor(props) {
|
||||
super(props);
|
||||
this.state = {
|
||||
recordSetsCPU: null,
|
||||
recordSetsFPGA: null,
|
||||
recordSetsIndex: 0,
|
||||
recordSetsLoaded: false,
|
||||
queryDescriptions: [
|
||||
{
|
||||
title: "Spark Job",
|
||||
sql:
|
||||
<ul>
|
||||
<li>
|
||||
taxi_input_df
|
||||
</li>
|
||||
<li>
|
||||
.groupBy('payment_type')
|
||||
.count()
|
||||
.select('payment_type', F.col('count').alias('total'))
|
||||
.show()
|
||||
</li>
|
||||
</ul>
|
||||
},
|
||||
{
|
||||
title: "Query 1",
|
||||
sql:
|
||||
<ul>
|
||||
<li>
|
||||
SELECT payment_type, COUNT(*) as total
|
||||
</li>
|
||||
<li>
|
||||
<ul>
|
||||
<li>
|
||||
FROM NewYorkYellowTaxi
|
||||
</li>
|
||||
<li>
|
||||
GROUP BY payment_type
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
},
|
||||
{
|
||||
title: "Query 2",
|
||||
sql:
|
||||
<ul>
|
||||
<li>
|
||||
SELECT passenger_count,
|
||||
</li>
|
||||
<li>
|
||||
AVG(total_amount) as avg_amount
|
||||
</li>
|
||||
<li>
|
||||
<ul>
|
||||
<li>
|
||||
FROM $tableName
|
||||
</li>
|
||||
<li>
|
||||
GROUP BY passenger_count
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
},
|
||||
{
|
||||
title: "Query 3",
|
||||
sql:
|
||||
<ul>
|
||||
<li>
|
||||
SELECT passenger_count,
|
||||
</li>
|
||||
<li>
|
||||
year(tpep_pickup_datetime) year,
|
||||
</li>
|
||||
<li>
|
||||
COUNT(*) total
|
||||
</li>
|
||||
<li>
|
||||
<ul>
|
||||
<li>
|
||||
FROM $tableName
|
||||
</li>
|
||||
<li>
|
||||
GROUP BY passenger_count, year
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
},
|
||||
],
|
||||
themes: [
|
||||
new Theme('themes', 'blue', '#0882D8', '#34c1ee', '#fff', '#8CDDF7', '#8cddf7', '#000', '#042246'),
|
||||
new Theme('themes', 'green', '#44683A', '#44683A', '#fff', '#82B74F', '#82B74F', '#000', '#042246'),
|
||||
new Theme('themes', 'green2', '#308950', '#308950', '#fff', '#80B173', '#80B173', '#000', '#042246'),
|
||||
new Theme('themes', 'magenta', '#E3008C', '#E3008C', '#fff', '#B4009E', '#B4009E', '#fff', '#042246'),
|
||||
new Theme('themes', 'orange', '#EB7415', '#EB7415', '#fff', '#F3AA6D', '#F3AA6D', '#000', '#042246'),
|
||||
new Theme('themes', 'purple', '#5059C8', '#5059C8', '#fff', '#7B83EB', '#7B83EB', '#fff', '#042246')
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
componentDidMount() {
|
||||
this.setState({
|
||||
recordSetsCPU: this.importRecordSets(cpu),
|
||||
recordSetsFPGA: this.importRecordSets(fpga)
|
||||
}, () => {
|
||||
this.setState({
|
||||
recordSetsLoaded: true
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
render() {
|
||||
return this.state.recordSetsLoaded ? (
|
||||
<Router>
|
||||
<Switch>
|
||||
<Route exact path="/">
|
||||
<React.Fragment>
|
||||
<Grid
|
||||
effects={true}
|
||||
scalar={1}
|
||||
themes={[
|
||||
this.state.themes[0]
|
||||
]}
|
||||
queryDescriptions={[
|
||||
this.state.queryDescriptions[0]
|
||||
]}
|
||||
recordSetsCPU={[
|
||||
this.state.recordSetsCPU[1]
|
||||
]}
|
||||
recordSetsFPGA={[
|
||||
this.state.recordSetsFPGA[0]
|
||||
]}
|
||||
/>
|
||||
</React.Fragment>
|
||||
</Route>
|
||||
</Switch>
|
||||
</Router>
|
||||
) : null
|
||||
}
|
||||
|
||||
importRecordSets = (dataSets) => {
|
||||
const recordSets = [];
|
||||
const dl = dataSets.length;
|
||||
for (let d = 0; d < dl; d++) {
|
||||
const recordSet = [];
|
||||
const dataSet = dataSets[d];
|
||||
const rl = dataSet.records.length;
|
||||
for (let r = 0; r < rl; r++) {
|
||||
recordSet.push(Record.fromData(dataSet.records[r]));
|
||||
}
|
||||
recordSets.push(recordSet);
|
||||
}
|
||||
return recordSets;
|
||||
}
|
||||
}
|
||||
|
||||
export default App;
|
|
@ -0,0 +1,9 @@
|
|||
import React from 'react';
|
||||
import { render } from '@testing-library/react';
|
||||
import App from './App';
|
||||
|
||||
test('renders learn react link', () => {
|
||||
const { getByText } = render(<App />);
|
||||
const linkElement = getByText(/learn react/i);
|
||||
expect(linkElement).toBeInTheDocument();
|
||||
});
|
|
@ -0,0 +1,27 @@
|
|||
import React from 'react';
|
||||
import { Meter } from './Meter';
|
||||
|
||||
export class CPU extends React.Component {
|
||||
render() {
|
||||
const theme = this.props.theme;
|
||||
return (
|
||||
<React.Fragment>
|
||||
<Meter
|
||||
effects={this.props.effects}
|
||||
spec={this.props.spec ? this.props.spec : "E16v3"}
|
||||
specColor={theme.cpuSpecColor}
|
||||
specShift={0}
|
||||
scalar={this.props.scalar}
|
||||
theme={theme}
|
||||
background={theme.cpu}
|
||||
color={theme.cpuColor}
|
||||
hoverColor={theme.cpuHoverColor}
|
||||
start={theme.startCPU}
|
||||
stop={theme.stopCPU}
|
||||
meterRange={this.props.meterRange}
|
||||
recordSet={this.props.recordSet}
|
||||
/>
|
||||
</React.Fragment>
|
||||
)
|
||||
}
|
||||
}
|
|
@ -0,0 +1,27 @@
|
|||
import React from 'react';
|
||||
import { Meter } from './Meter';
|
||||
|
||||
export class FPGA extends React.Component {
|
||||
render() {
|
||||
const theme = this.props.theme;
|
||||
return (
|
||||
<React.Fragment>
|
||||
<Meter
|
||||
effects={this.props.effects}
|
||||
spec={this.props.spec ? this.props.spec : "NP10"}
|
||||
specColor={theme.fpgaSpecColor}
|
||||
specShift={5}
|
||||
scalar={this.props.scalar}
|
||||
theme={theme}
|
||||
background={theme.fpga}
|
||||
color={theme.fpgaColor}
|
||||
hoverColor={theme.fpgaHoverColor}
|
||||
start={theme.startFPGA}
|
||||
stop={theme.stopFPGA}
|
||||
meterRange={this.props.meterRange}
|
||||
recordSet={this.props.recordSet}
|
||||
/>
|
||||
</React.Fragment>
|
||||
)
|
||||
}
|
||||
}
|