Merge branch 'dev' into temp
# Conflicts: # examples/config/es_settings.json # examples/config/es_supervisor.conf # examples/etl.py # spot/spot_manager.py
This commit is contained in:
Kyle Lahnakoski
Коммит
bcc57f4ff3
|
|
@ -3,7 +3,6 @@
|
|||
/pyLibrary/.svn
|
||||
*.pyc
|
||||
/results
|
||||
/resources/aws/prices.json
|
||||
/examples/config/etl_supervisor.conf.alt
|
||||
/result
|
||||
/tests
|
||||
|
|
|
|||
13
README.md
13
README.md
|
|
@ -11,7 +11,7 @@ The module assumes your workload is **long running** and has
|
|||
**many save-points**.
|
||||
|
||||
In my case each machine is setup to pull small tasks off a queue and
|
||||
execute them. These machines can be shutdown at any time; with the most
|
||||
execute them. These machines can be shutdown at any time; with the most
|
||||
recent task simply placed back on the queue for some other machine to run.
|
||||
|
||||
## Overview
|
||||
|
|
@ -32,7 +32,7 @@ with the best `estimated_value`, are bid on first.
|
|||
* boto
|
||||
* requests
|
||||
* ecdsa (required by fabric, but not installed by pip)
|
||||
* fabric
|
||||
* fabric2
|
||||
|
||||
## Installation
|
||||
|
||||
|
|
@ -45,7 +45,7 @@ For now, you must clone the repo
|
|||
There are three main branches
|
||||
|
||||
* **dev** - development done here (unstable)
|
||||
* **beta** - not used
|
||||
* **manager-etl** - multithreaded management, not ready for ES node management (Oct 2018)
|
||||
* **manager** - used to manage the staging clusters
|
||||
* **master** - proven stable on **manager** for at least a few days
|
||||
|
||||
|
|
@ -162,7 +162,7 @@ Some caveats:
|
|||
ephemeral drives, but the EBS will be removed too. If you want the volume
|
||||
to be permanent, you must map the block device yourself.
|
||||
* ***block devices will not be formatted nor mounted***. The `path` is
|
||||
provided only so the InstanceManger.setup() routine can perform the `mkfs`
|
||||
provided only so the `InstanceManger.setup()` routine can perform the `mkfs`
|
||||
and `mount` commands.
|
||||
|
||||
### Writing a InstanceManager
|
||||
|
|
@ -177,9 +177,10 @@ also up to you. The `examples` uses the size of the pending queue to
|
|||
determine, roughly, how much utility is required.
|
||||
* **`setup()`** - function is called to setup an instance. It is passed
|
||||
both a boto ec2 instance object, and the utility this instance is
|
||||
expected to provide.
|
||||
expected to provide. This is run in its own thread, and multiple can be
|
||||
called at the same time; ensure your code is threadsafe.
|
||||
* **`teardown()`** - When the machine is no longer required, this will be
|
||||
called before SpotManager terminates the EC2 instance. This method is
|
||||
called before SpotManager terminates the EC2 instance. This method is
|
||||
*not* called when AWS terminates the instance.
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ path.logs: /data1/logs
|
|||
|
||||
discovery.ec2.endpoint: ec2.us-west-2.amazonaws.com
|
||||
discovery.zen.hosts_provider: ec2
|
||||
discovery.zen.minimum_master_nodes: 1
|
||||
discovery.zen.minimum_master_nodes: 2
|
||||
|
||||
http.compression: true
|
||||
http.cors.enabled: true
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"budget": 3.00, //MAXIMUM SPEND PER HOUR FOR ALL INSTANCES
|
||||
"budget": 4.00, //MAXIMUM SPEND PER HOUR FOR ALL INSTANCES
|
||||
"max_utility_price": 0.02, //MOST THAT WILL BE SPENT ON A SINGLE UTILITY POINT
|
||||
"max_new_utility": 120, //MOST NEW UTILITY THAT WILL BE REQUESTED IN A SINGLE RUN
|
||||
"max_requests_per_type": 2, //LIMIT THE NUMBER OF NET-NEW REQUESTS BY TYPE
|
||||
|
|
@ -19,6 +19,44 @@
|
|||
"more_drives":[
|
||||
{"path":"/data1", "size":1000, "volume_type":"standard"}
|
||||
],
|
||||
"1_nvm_drives":[
|
||||
{"path":"/data1", "device":"/dev/nvme0n1"}
|
||||
],
|
||||
"2_nvm_drives":[
|
||||
{"path":"/data1", "device":"/dev/nvme0n1"},
|
||||
{"path":"/data2", "device":"/dev/nvme1n1"}
|
||||
],
|
||||
"3_nvm_drives":[
|
||||
{"path":"/data1", "device":"/dev/nvme0n1"},
|
||||
{"path":"/data2", "device":"/dev/nvme1n1"},
|
||||
{"path":"/data3", "device":"/dev/nvme2n1"}
|
||||
],
|
||||
"4_nvm_drives":[
|
||||
{"path":"/data1", "device":"/dev/nvme0n1"},
|
||||
{"path":"/data2", "device":"/dev/nvme1n1"},
|
||||
{"path":"/data3", "device":"/dev/nvme2n1"},
|
||||
{"path":"/data4", "device":"/dev/nvme3n1"}
|
||||
],
|
||||
"6_nvm_drives":[
|
||||
{"path":"/data1", "device":"/dev/nvme0n1"},
|
||||
{"path":"/data2", "device":"/dev/nvme1n1"},
|
||||
{"path":"/data3", "device":"/dev/nvme2n1"},
|
||||
{"path":"/data4", "device":"/dev/nvme3n1"},
|
||||
{"path":"/data5", "device":"/dev/nvme4n1"},
|
||||
{"path":"/data6", "device":"/dev/nvme5n1"}
|
||||
],
|
||||
"8_nvm_drives":[
|
||||
{"path":"/data1", "device":"/dev/nvme0n1"},
|
||||
{"path":"/data2", "device":"/dev/nvme1n1"},
|
||||
{"path":"/data3", "device":"/dev/nvme2n1"},
|
||||
{"path":"/data4", "device":"/dev/nvme3n1"},
|
||||
{"path":"/data5", "device":"/dev/nvme4n1"},
|
||||
{"path":"/data6", "device":"/dev/nvme5n1"},
|
||||
{"path":"/data7", "device":"/dev/nvme6n1"},
|
||||
{"path":"/data8", "device":"/dev/nvme7n1"}
|
||||
],
|
||||
|
||||
|
||||
"1_ephemeral_drives":[
|
||||
{"path":"/data1", "device":"/dev/sdb"}
|
||||
],
|
||||
|
|
@ -57,59 +95,97 @@
|
|||
],
|
||||
"utility":[
|
||||
// ONE POINT PER 1 GIG OF MEMORY. OR 2 PER 100 GIG OF DRIVESPACE, OR 60 POINTS, WHICHEVER IS LESS
|
||||
// EBS IS WAY TO SLOW FOR ELASTICSEARCH
|
||||
// {"instance_type": "c1.medium", "storage": 350, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 5, "num_drives": 1, "memory": 1.7, "cpu": 2, "utility": 1.7},
|
||||
// {"instance_type": "c1.xlarge", "storage": 1680, "drives": {"$ref": "#4_ephemeral_drives"}, "discount": 0, "ecu": 20, "num_drives": 4, "memory": 7, "cpu": 8, "utility": 7},
|
||||
{"instance_type": "c3.2xlarge", "storage": 160, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 28, "num_drives": 2, "memory": 15, "cpu": 8, "utility": 3.2},
|
||||
{"instance_type": "c3.4xlarge", "storage": 320, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 55, "num_drives": 2, "memory": 30, "cpu": 16, "utility": 6.4},
|
||||
{"instance_type": "c3.8xlarge", "storage": 640, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 108, "num_drives": 2, "memory": 60, "cpu": 32, "utility": 12.8},
|
||||
{"instance_type": "c3.large", "storage": 32, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 7, "num_drives": 2, "memory": 3.75, "cpu": 2, "utility": 0.64},
|
||||
{"instance_type": "c3.xlarge", "storage": 80, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 14, "num_drives": 2, "memory": 7.5, "cpu": 4, "utility": 1.6},
|
||||
{"blacklist":true, "instance_type": "cc2.8xlarge", "storage": 3360, "drives": {"$ref": "#4_ephemeral_drives"}, "discount": 0, "ecu": 88, "num_drives": 4, "memory": 60.5, "cpu": 32, "utility": 60},
|
||||
// {"instance_type": "cg1.4xlarge", "storage": 1680, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 33.5, "num_drives": 2, "memory": 22.5, "cpu": 16, "utility": 22.5},
|
||||
{"instance_type": "cr1.8xlarge", "storage": 240, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 88, "num_drives": 2, "memory": 244, "cpu": 32, "utility": 4.8},
|
||||
{"instance_type": "d2.2xlarge", "storage": 12000, "drives": {"$ref": "#6_ephemeral_drives"}, "discount": 0, "ecu": 28, "num_drives": 6, "memory": 61, "cpu": 8, "utility": 60},
|
||||
{"instance_type": "d2.4xlarge", "storage": 24000, "drives": {"$ref": "#8_ephemeral_drives"}, "discount": 0, "ecu": 56, "num_drives": 12, "memory": 122, "cpu": 16, "utility": 60},
|
||||
{"instance_type": "d2.8xlarge", "storage": 48000, "drives": {"$ref": "#8_ephemeral_drives"}, "discount": 0, "ecu": 116, "num_drives": 24, "memory": 244, "cpu": 36, "utility": 60},
|
||||
{"instance_type": "d2.xlarge", "storage": 6000, "drives": {"$ref": "#3_ephemeral_drives"}, "discount": 0, "ecu": 14, "num_drives": 3, "memory": 30.5, "cpu": 4, "utility": 30.5},
|
||||
{"instance_type": "g2.2xlarge", "storage": 60, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 26, "num_drives": 1, "memory": 15, "cpu": 8, "utility": 1.2},
|
||||
{"instance_type": "g2.8xlarge", "storage": 240, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 104, "num_drives": 2, "memory": 60, "cpu": 32, "utility": 4.8},
|
||||
{"instance_type": "hi1.4xlarge", "storage": 2048, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 35, "num_drives": 2, "memory": 60.5, "cpu": 16, "utility": 40.96},
|
||||
// {"instance_type": "hs1.8xlarge", "storage": 48000, "drives": {"$ref": "#8_ephemeral_drives"}, "discount": 0, "ecu": 35, "num_drives": 24, "memory": 117, "cpu": 16, "utility": 60},
|
||||
{"instance_type": "i2.2xlarge", "storage": 1600, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 27, "num_drives": 2, "memory": 61, "cpu": 8, "utility": 32.0},
|
||||
{"instance_type": "i2.4xlarge", "storage": 3200, "drives": {"$ref": "#4_ephemeral_drives"}, "discount": 0, "ecu": 53, "num_drives": 4, "memory": 122, "cpu": 16, "utility": 60},
|
||||
{"instance_type": "i2.8xlarge", "storage": 6400, "drives": {"$ref": "#8_ephemeral_drives"}, "discount": 0, "ecu": 104, "num_drives": 8, "memory": 244, "cpu": 32, "utility": 60},
|
||||
{"instance_type": "i2.xlarge", "storage": 800, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 14, "num_drives": 1, "memory": 30.5, "cpu": 4, "utility": 16.0},
|
||||
// {"instance_type": "i3.16xlarge", "storage": 15200, "drives": {"$ref": "#8_ephemeral_drives"}, "discount": 0, "ecu": 200, "num_drives": 8, "memory": 488, "cpu": 64, "utility": 60},
|
||||
// {"instance_type": "i3.2xlarge", "storage": 1900, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 27, "num_drives": 1, "memory": 61, "cpu": 8, "utility": 38.0},
|
||||
// {"instance_type": "i3.4xlarge", "storage": 3800, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 53, "num_drives": 2, "memory": 122, "cpu": 16, "utility": 60},
|
||||
// {"instance_type": "i3.8xlarge", "storage": 7600, "drives": {"$ref": "#4_ephemeral_drives"}, "discount": 0, "ecu": 99, "num_drives": 4, "memory": 244, "cpu": 32, "utility": 60},
|
||||
// {"instance_type": "i3.large", "storage": 475, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 7, "num_drives": 1, "memory": 15.25, "cpu": 2, "utility": 9.5},
|
||||
// {"instance_type": "i3.xlarge", "storage": 950, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 13, "num_drives": 1, "memory": 30.5, "cpu": 4, "utility": 19.0},
|
||||
// {"instance_type": "m1.large", "storage": 840, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 4, "num_drives": 2, "memory": 7.5, "cpu": 2, "utility": 7.5},
|
||||
// {"instance_type": "m1.medium", "storage": 410, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 2, "num_drives": 1, "memory": 3.75, "cpu": 1, "utility": 3.75},
|
||||
// {"instance_type": "m1.small", "storage": 160, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 1, "num_drives": 1, "memory": 1.7, "cpu": 1, "utility": 1.7},
|
||||
// {"instance_type": "m1.xlarge", "storage": 1680, "drives": {"$ref": "#4_ephemeral_drives"}, "discount": 0, "ecu": 8, "num_drives": 4, "memory": 15, "cpu": 4, "utility": 15},
|
||||
// {"instance_type": "m2.2xlarge", "storage": 850, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 13, "num_drives": 1, "memory": 34.2, "cpu": 4, "utility": 17.0},
|
||||
// {"instance_type": "m2.4xlarge", "storage": 1680, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 26, "num_drives": 2, "memory": 68.4, "cpu": 8, "utility": 33.6},
|
||||
// {"instance_type": "m2.xlarge", "storage": 420, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 6.5, "num_drives": 1, "memory": 17.1, "cpu": 2, "utility": 8.4},
|
||||
{"instance_type": "m3.2xlarge", "storage": 160, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 26, "num_drives": 2, "memory": 30, "cpu": 8, "utility": 3.2},
|
||||
{"instance_type": "m3.large", "storage": 32, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 6.5, "num_drives": 1, "memory": 7.5, "cpu": 2, "utility": 0.64},
|
||||
{"instance_type": "m3.medium", "storage": 4, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 3, "num_drives": 1, "memory": 3.75, "cpu": 1, "utility": 0.08},
|
||||
{"instance_type": "m3.xlarge", "storage": 80, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 13, "num_drives": 2, "memory": 15, "cpu": 4, "utility": 1.6},
|
||||
{"instance_type": "r3.2xlarge", "storage": 160, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 26, "num_drives": 1, "memory": 61, "cpu": 8, "utility": 3.2},
|
||||
{"instance_type": "r3.4xlarge", "storage": 320, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 52, "num_drives": 1, "memory": 122, "cpu": 16, "utility": 6.4},
|
||||
{"instance_type": "r3.8xlarge", "storage": 640, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 104, "num_drives": 2, "memory": 244, "cpu": 32, "utility": 12.8},
|
||||
{"instance_type": "r3.large", "storage": 32, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 6.5, "num_drives": 1, "memory": 15.25, "cpu": 2, "utility": 0.64},
|
||||
{"instance_type": "r3.xlarge", "storage": 80, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 13, "num_drives": 1, "memory": 30.5, "cpu": 4, "utility": 1.6},
|
||||
{"instance_type": "x1.16xlarge", "storage": 1920, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 174.5, "num_drives": 1, "memory": 976, "cpu": 64, "utility": 38.4},
|
||||
{"instance_type": "x1.32xlarge", "storage": 3840, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 349, "num_drives": 2, "memory": 1952, "cpu": 128, "utility": 60}
|
||||
// EBS IS TOO SLOW FOR ELASTICSEARCH
|
||||
|
||||
// {"instance_type": "c3.large", "storage": 32, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "num_drives": 2, "memory": 3.75, "cpu": 2, "utility": 0.64},
|
||||
{"instance_type": "c3.xlarge", "storage": 80, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "num_drives": 2, "memory": 7.5, "cpu": 4, "utility": 1.6},
|
||||
{"instance_type": "c3.2xlarge", "storage": 160, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "num_drives": 2, "memory": 15, "cpu": 8, "utility": 3.2},
|
||||
{"instance_type": "c3.4xlarge", "storage": 320, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "num_drives": 2, "memory": 30, "cpu": 16, "utility": 6.4},
|
||||
{"instance_type": "c3.8xlarge", "storage": 640, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "num_drives": 2, "memory": 60, "cpu": 32, "utility": 12.8},
|
||||
|
||||
{"instance_type": "cc2.8xlarge", "storage": 3360, "drives": {"$ref": "#4_ephemeral_drives"}, "discount": 0, "num_drives": 4, "memory": 60.5, "cpu": 32, "utility": 60},
|
||||
{"instance_type": "cr1.8xlarge", "storage": 240, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "num_drives": 2, "memory": 244, "cpu": 32, "utility": 4.8},
|
||||
|
||||
{"instance_type": "d2.xlarge", "storage": 6000, "drives": {"$ref": "#3_ephemeral_drives"}, "discount": 0, "num_drives": 3, "memory": 30.5, "cpu": 4, "utility": 30.5},
|
||||
{"instance_type": "d2.2xlarge", "storage": 12000, "drives": {"$ref": "#6_ephemeral_drives"}, "discount": 0, "num_drives": 6, "memory": 61, "cpu": 8, "utility": 60},
|
||||
{"instance_type": "d2.4xlarge", "storage": 24000, "drives": {"$ref": "#8_ephemeral_drives"}, "discount": 0, "num_drives": 12, "memory": 122, "cpu": 16, "utility": 60},
|
||||
{"instance_type": "d2.8xlarge", "storage": 48000, "drives": {"$ref": "#8_ephemeral_drives"}, "discount": 0, "num_drives": 24, "memory": 244, "cpu": 36, "utility": 60},
|
||||
|
||||
{"instance_type": "f1.2xlarge", "storage": 470, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 122, "cpu": 8, "utility": 9.4},
|
||||
{"instance_type": "f1.4xlarge", "storage": 940, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 244, "cpu": 16, "utility": 18.8},
|
||||
{"instance_type": "f1.16xlarge", "storage": 3760, "drives": {"$ref": "#4_ephemeral_drives"}, "discount": 0, "num_drives": 4, "memory": 976, "cpu": 64, "utility": 60},
|
||||
|
||||
{"instance_type": "g2.2xlarge", "storage": 60, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 15, "cpu": 8, "utility": 1.2},
|
||||
{"instance_type": "g2.8xlarge", "storage": 240, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "num_drives": 2, "memory": 60, "cpu": 32, "utility": 4.8},
|
||||
|
||||
{"instance_type": "h1.2xlarge", "storage": 2000, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 32, "cpu": 8, "utility": 32},
|
||||
{"instance_type": "h1.4xlarge", "storage": 4000, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "num_drives": 2, "memory": 64, "cpu": 16, "utility": 60},
|
||||
{"instance_type": "h1.8xlarge", "storage": 8000, "drives": {"$ref": "#4_ephemeral_drives"}, "discount": 0, "num_drives": 4, "memory": 128, "cpu": 32, "utility": 60},
|
||||
{"instance_type": "h1.16xlarge", "storage": 16000, "drives": {"$ref": "#8_ephemeral_drives"}, "discount": 0, "num_drives": 8, "memory": 256, "cpu": 64, "utility": 60},
|
||||
|
||||
{"instance_type": "hs1.8xlarge", "storage": 48000, "drives": {"$ref": "#8_ephemeral_drives"}, "discount": 0, "num_drives": 24, "memory": 117, "cpu": 16, "utility": 60},
|
||||
|
||||
{"instance_type": "i2.xlarge", "storage": 800, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 30.5, "cpu": 4, "utility": 16.0},
|
||||
{"instance_type": "i2.2xlarge", "storage": 1600, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "num_drives": 2, "memory": 61, "cpu": 8, "utility": 32.0},
|
||||
{"instance_type": "i2.4xlarge", "storage": 3200, "drives": {"$ref": "#4_ephemeral_drives"}, "discount": 0, "num_drives": 4, "memory": 122, "cpu": 16, "utility": 60},
|
||||
{"instance_type": "i2.8xlarge", "storage": 6400, "drives": {"$ref": "#8_ephemeral_drives"}, "discount": 0, "num_drives": 8, "memory": 244, "cpu": 32, "utility": 60},
|
||||
|
||||
//PROBLEM CONNECTING EPHEMERAL DRIVES
|
||||
{"instance_type": "i3.large", "storage": 475, "drives": {"$ref": "#1_nvm_drives"}, "discount": 0, "num_drives": 1, "memory": 15.25, "cpu": 2, "utility": 9.5},
|
||||
{"instance_type": "i3.xlarge", "storage": 950, "drives": {"$ref": "#1_nvm_drives"}, "discount": 0, "num_drives": 1, "memory": 30.5, "cpu": 4, "utility": 19.0},
|
||||
{"instance_type": "i3.2xlarge", "storage": 1900, "drives": {"$ref": "#1_nvm_drives"}, "discount": 0, "num_drives": 1, "memory": 61, "cpu": 8, "utility": 38.0},
|
||||
{"instance_type": "i3.4xlarge", "storage": 3800, "drives": {"$ref": "#2_nvm_drives"}, "discount": 0, "num_drives": 2, "memory": 122, "cpu": 16, "utility": 60},
|
||||
{"instance_type": "i3.8xlarge", "storage": 7600, "drives": {"$ref": "#4_nvm_drives"}, "discount": 0, "num_drives": 4, "memory": 244, "cpu": 32, "utility": 60},
|
||||
{"instance_type": "i3.16xlarge", "storage": 15200, "drives": {"$ref": "#8_nvm_drives"}, "discount": 0, "num_drives": 8, "memory": 488, "cpu": 64, "utility": 60},
|
||||
|
||||
// NO CAPACITY
|
||||
{"instance_type": "m5d.large", "storage": 75, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 8, "cpu": 2, "utility": 1.5},
|
||||
{"instance_type": "m5d.xlarge", "storage": 150, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 16, "cpu": 4, "utility": 3},
|
||||
{"instance_type": "m5d.2xlarge", "storage": 300, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 32, "cpu": 8, "utility": 6},
|
||||
{"instance_type": "m5d.4xlarge", "storage": 600, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "num_drives": 2, "memory": 64, "cpu": 16, "utility": 12},
|
||||
{"instance_type": "m5d.12xlarge", "storage": 1800, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "num_drives": 2, "memory": 192, "cpu": 48, "utility": 36},
|
||||
{"instance_type": "m5d.24xlarge", "storage": 3600, "drives": {"$ref": "#4_ephemeral_drives"}, "discount": 0, "num_drives": 4, "memory": 384, "cpu": 96, "utility": 60},
|
||||
|
||||
//NO CAPACTIY
|
||||
{"instance_type": "r5d.large", "storage": 75, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 16, "cpu": 2, "utility": 1.5},
|
||||
{"instance_type": "r5d.xlarge", "storage": 150, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 32, "cpu": 4, "utility": 3},
|
||||
{"instance_type": "r5d.2xlarge", "storage": 300, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 64, "cpu": 8, "utility": 6},
|
||||
{"instance_type": "r5d.4xlarge", "storage": 600, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "num_drives": 2, "memory": 128, "cpu": 16, "utility": 12},
|
||||
{"instance_type": "r5d.12xlarge", "storage": 1800, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "num_drives": 2, "memory": 384, "cpu": 48, "utility": 36},
|
||||
{"instance_type": "r5d.24xlarge", "storage": 3600, "drives": {"$ref": "#4_ephemeral_drives"}, "discount": 0, "num_drives": 4, "memory": 768, "cpu": 96, "utility": 60},
|
||||
|
||||
{"instance_type": "c5d.large", "storage": 50, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 4, "cpu": 2, "utility": 1},
|
||||
{"instance_type": "c5d.xlarge", "storage": 100, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 8, "cpu": 4, "utility": 2},
|
||||
{"instance_type": "c5d.2xlarge", "storage": 200, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 16, "cpu": 8, "utility": 4},
|
||||
{"instance_type": "c5d.4xlarge", "storage": 400, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 32, "cpu": 16, "utility": 8},
|
||||
{"instance_type": "c5d.9xlarge", "storage": 900, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 72, "cpu": 36, "utility": 18},
|
||||
{"instance_type": "c5d.18xlarge", "storage": 1800, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "num_drives": 2, "memory": 144, "cpu": 72, "utility": 36},
|
||||
|
||||
{"instance_type": "x1.16xlarge", "storage": 1920, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 976, "cpu": 64, "utility": 38.4},
|
||||
{"instance_type": "x1.32xlarge", "storage": 3840, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "num_drives": 2, "memory": 1952, "cpu": 128, "utility": 60},
|
||||
|
||||
{"instance_type": "x1e.xlarge", "storage": 120, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 122, "cpu": 4, "utility": 2.4},
|
||||
{"instance_type": "x1e.2xlarge", "storage": 240, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 244, "cpu": 8, "utility": 4.8},
|
||||
{"instance_type": "x1e.4xlarge", "storage": 480, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 488, "cpu": 16, "utility": 9.6},
|
||||
{"instance_type": "x1e.8xlarge", "storage": 960, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 976, "cpu": 32, "utility": 19.2},
|
||||
{"instance_type": "x1e.16xlarge", "storage": 1920, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 1952, "cpu": 64, "utility": 38.4},
|
||||
{"instance_type": "x1e.32xlarge", "storage": 3840, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "num_drives": 2, "memory": 3904, "cpu": 128, "utility": 60},
|
||||
|
||||
{"instance_type": "z1d.large", "storage": 75, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 16, "cpu": 2, "utility": 1.5},
|
||||
{"instance_type": "z1d.xlarge", "storage": 150, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 32, "cpu": 4, "utility": 3},
|
||||
{"instance_type": "z1d.2xlarge", "storage": 300, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 64, "cpu": 8, "utility": 6},
|
||||
{"instance_type": "z1d.3xlarge", "storage": 450, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 96, "cpu": 12, "utility": 9},
|
||||
{"instance_type": "z1d.6xlarge", "storage": 900, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 192, "cpu": 24, "utility": 18},
|
||||
{"instance_type": "z1d.12xlarge", "storage": 1800, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "num_drives": 2, "memory": 384, "cpu": 48, "utility": 36}
|
||||
|
||||
|
||||
|
||||
],
|
||||
"ec2": {
|
||||
"request": {
|
||||
//SEE http://boto.readthedocs.org/en/latest/ref/ec2.html#boto.ec2.connection.EC2Connection.request_spot_instances
|
||||
"price": 0.001,
|
||||
"image_id": "ami-bf4193c7",
|
||||
"image_id": "ami-01bbe152bf19d0289", // "ami-bf4193c7",
|
||||
"count": 1,
|
||||
"type": "one-time",
|
||||
"valid_from": null,
|
||||
|
|
@ -166,6 +242,9 @@
|
|||
"encrypted":false
|
||||
}
|
||||
},
|
||||
"constants":{
|
||||
"mo_http.http.default_headers":{"Referer": "https://github.com/klahnakoski/SpotManager"}
|
||||
},
|
||||
"debug": {
|
||||
"trace": true,
|
||||
"cprofile": {
|
||||
|
|
@ -180,6 +259,12 @@
|
|||
"backupCount": 10,
|
||||
"encoding": "utf8"
|
||||
},
|
||||
{
|
||||
"log_type": "elasticsearch",
|
||||
"host": "http://activedata.allizom.org",
|
||||
"index": "debug-spot-manager",
|
||||
"timeout": 600
|
||||
},
|
||||
{
|
||||
"log_type": "ses",
|
||||
"from_address": "klahnakoski@mozilla.com",
|
||||
|
|
|
|||
|
|
@ -1,46 +0,0 @@
|
|||
cluster.name: active-data
|
||||
node.zone: spot
|
||||
node.name: spot_{{id}}
|
||||
node.master: false
|
||||
node.data: true
|
||||
|
||||
script.inline: on
|
||||
script.indexed: on
|
||||
|
||||
|
||||
cluster.routing.allocation.cluster_concurrent_rebalance: 1
|
||||
cluster.routing.allocation.node_concurrent_recoveries: 1
|
||||
|
||||
bootstrap.mlockall: true
|
||||
path.data: {{data_paths}}
|
||||
path.logs: /data1/logs
|
||||
cloud:
|
||||
aws:
|
||||
region: us-west-2
|
||||
protocol: https
|
||||
ec2:
|
||||
protocol: https
|
||||
discovery.type: ec2
|
||||
discovery.zen.ping.multicast.enabled: false
|
||||
discovery.zen.minimum_master_nodes: 1
|
||||
|
||||
index.number_of_shards: 1
|
||||
index.number_of_replicas: 1
|
||||
index.cache.field.type: soft
|
||||
index.translog.interval: 60s
|
||||
index.translog.flush_threshold_size: 1gb
|
||||
|
||||
indices.memory.index_buffer_size: 20%
|
||||
indices.recovery.concurrent_streams: 1
|
||||
indices.recovery.max_bytes_per_sec: 1000mb
|
||||
indices.store.throttle.type: none
|
||||
|
||||
http.compression: true
|
||||
http.cors.allow-origin: "/.*/"
|
||||
http.cors.enabled: true
|
||||
http.compression: true
|
||||
http.max_content_length: 1000mb
|
||||
http.timeout: 600
|
||||
|
||||
threadpool.bulk.queue_size: 3000
|
||||
threadpool.index.queue_size: 1000
|
||||
|
|
@ -1,64 +0,0 @@
|
|||
#!/bin/sh
|
||||
|
||||
ES_CLASSPATH=$ES_CLASSPATH:$ES_HOME/lib/elasticsearch-1.7.1.jar:$ES_HOME/lib/*:$ES_HOME/lib/sigar/*
|
||||
|
||||
ES_MIN_MEM={{memory}}g
|
||||
ES_MAX_MEM={{memory}}g
|
||||
|
||||
# min and max heap sizes should be set to the same value to avoid
|
||||
# stop-the-world GC pauses during resize, and so that we can lock the
|
||||
# heap in memory on startup to prevent any of it from being swapped
|
||||
# out.
|
||||
JAVA_OPTS="$JAVA_OPTS -Xms${ES_MIN_MEM}"
|
||||
JAVA_OPTS="$JAVA_OPTS -Xmx${ES_MAX_MEM}"
|
||||
|
||||
# new generation
|
||||
if [ "x$ES_HEAP_NEWSIZE" != "x" ]; then
|
||||
JAVA_OPTS="$JAVA_OPTS -Xmn${ES_HEAP_NEWSIZE}"
|
||||
fi
|
||||
|
||||
# max direct memory
|
||||
if [ "x$ES_DIRECT_SIZE" != "x" ]; then
|
||||
JAVA_OPTS="$JAVA_OPTS -XX:MaxDirectMemorySize=${ES_DIRECT_SIZE}"
|
||||
fi
|
||||
|
||||
# reduce the per-thread stack size
|
||||
JAVA_OPTS="$JAVA_OPTS -Xss256k"
|
||||
|
||||
# set to headless, just in case
|
||||
JAVA_OPTS="$JAVA_OPTS -Djava.awt.headless=true"
|
||||
|
||||
# Force the JVM to use IPv4 stack
|
||||
if [ "x$ES_USE_IPV4" != "x" ]; then
|
||||
JAVA_OPTS="$JAVA_OPTS -Djava.net.preferIPv4Stack=true"
|
||||
fi
|
||||
|
||||
JAVA_OPTS="$JAVA_OPTS -XX:+UseParNewGC"
|
||||
JAVA_OPTS="$JAVA_OPTS -XX:+UseConcMarkSweepGC"
|
||||
|
||||
JAVA_OPTS="$JAVA_OPTS -XX:CMSInitiatingOccupancyFraction=75"
|
||||
JAVA_OPTS="$JAVA_OPTS -XX:+UseCMSInitiatingOccupancyOnly"
|
||||
|
||||
# GC logging options
|
||||
if [ "x$ES_USE_GC_LOGGING" != "x" ]; then
|
||||
JAVA_OPTS="$JAVA_OPTS -XX:+PrintGCDetails"
|
||||
JAVA_OPTS="$JAVA_OPTS -XX:+PrintGCTimeStamps"
|
||||
JAVA_OPTS="$JAVA_OPTS -XX:+PrintClassHistogram"
|
||||
JAVA_OPTS="$JAVA_OPTS -XX:+PrintTenuringDistribution"
|
||||
JAVA_OPTS="$JAVA_OPTS -XX:+PrintGCApplicationStoppedTime"
|
||||
JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/elasticsearch/gc.log"
|
||||
fi
|
||||
|
||||
# Causes the JVM to dump its heap on OutOfMemory.
|
||||
JAVA_OPTS="$JAVA_OPTS -XX:+HeapDumpOnOutOfMemoryError"
|
||||
# The path to the heap dump location, note directory must exists and have enough
|
||||
# space for a full heap dump.
|
||||
JAVA_OPTS="$JAVA_OPTS -XX:HeapDumpPath=/data/heapdump/heapdump.hprof"
|
||||
|
||||
# Disables explicit GC
|
||||
JAVA_OPTS="$JAVA_OPTS -XX:+DisableExplicitGC"
|
||||
|
||||
# Ensure UTF-8 encoding by default (e.g. filenames)
|
||||
JAVA_OPTS="$JAVA_OPTS -Dfile.encoding=UTF-8"
|
||||
|
||||
|
||||
|
|
@ -1,195 +0,0 @@
|
|||
{
|
||||
"budget": 4.00, //MAXIMUM SPEND PER HOUR FOR ALL INSTANCES
|
||||
"max_utility_price": 0.02, //MOST THAT WILL BE SPENT ON A SINGLE UTILITY POINT
|
||||
"max_new_utility": 120, //MOST NEW UTILITY THAT WILL BE REQUESTED IN A SINGLE RUN
|
||||
"max_requests_per_type": 2, //LIMIT THE NUMBER OF NET-NEW REQUESTS BY TYPE
|
||||
"max_percent_per_type": 0.50, //ALL INSTANCE TYPES MAY NOT GO OVER THIS AS A PERCENT OF TOTAL INSTANCES (USED TO MITIGATE LOOSING ALL INSTANCES AT ONCE)
|
||||
"uptime":{
|
||||
"history": "week", //HOW MUCH HISTORY TO USE
|
||||
"duration": "day", //HOW LONG WE WOULD LIKE OUR MACHINE TO TO STAY UP
|
||||
"bid_percentile": 0.95 //THE PROBABILITY WE ACHIEVE OUR UPTIME
|
||||
},
|
||||
"price_file": "resources/aws/prices.json",
|
||||
"run_interval": "10minute", //HOW LONG BEFORE NEXT RUN
|
||||
"availability_zone": "us-west-2c",
|
||||
"product":"Linux/UNIX (Amazon VPC)",
|
||||
"aws": {
|
||||
"$ref": "//~/private.json#aws_credentials"
|
||||
},
|
||||
"more_drives":[
|
||||
{"path":"/data1", "size":1000, "volume_type":"standard"}
|
||||
],
|
||||
"1_ephemeral_drives":[
|
||||
{"path":"/data1", "device":"/dev/sdb"}
|
||||
],
|
||||
"2_ephemeral_drives":[
|
||||
{"path":"/data1", "device":"/dev/sdb"},
|
||||
{"path":"/data2", "device":"/dev/sdc"}
|
||||
],
|
||||
"3_ephemeral_drives":[
|
||||
{"path":"/data1", "device":"/dev/sdb"},
|
||||
{"path":"/data2", "device":"/dev/sdc"},
|
||||
{"path":"/data3", "device":"/dev/sdd"}
|
||||
],
|
||||
"4_ephemeral_drives":[
|
||||
{"path":"/data1", "device":"/dev/sdb"},
|
||||
{"path":"/data2", "device":"/dev/sdc"},
|
||||
{"path":"/data3", "device":"/dev/sdd"},
|
||||
{"path":"/data4", "device":"/dev/sde"}
|
||||
],
|
||||
"6_ephemeral_drives":[
|
||||
{"path":"/data1", "device":"/dev/sdb"},
|
||||
{"path":"/data2", "device":"/dev/sdc"},
|
||||
{"path":"/data3", "device":"/dev/sdd"},
|
||||
{"path":"/data4", "device":"/dev/sde"},
|
||||
{"path":"/data5", "device":"/dev/sdf"},
|
||||
{"path":"/data6", "device":"/dev/sdg"}
|
||||
],
|
||||
"8_ephemeral_drives":[
|
||||
{"path":"/data1", "device":"/dev/sdb"},
|
||||
{"path":"/data2", "device":"/dev/sdc"},
|
||||
{"path":"/data3", "device":"/dev/sdd"},
|
||||
{"path":"/data4", "device":"/dev/sde"},
|
||||
{"path":"/data5", "device":"/dev/sdf"},
|
||||
{"path":"/data6", "device":"/dev/sdg"},
|
||||
{"path":"/data7", "device":"/dev/sdh"},
|
||||
{"path":"/data8", "device":"/dev/sdi"}
|
||||
],
|
||||
"utility":[
|
||||
// ONE POINT PER 1 GIG OF MEMORY. OR 2 PER 100 GIG OF DRIVESPACE, OR 60 POINTS, WHICHEVER IS LESS
|
||||
// EBS IS WAY TO SLOW FOR ELASTICSEARCH
|
||||
// {"instance_type": "c1.medium", "storage": 350, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 5, "num_drives": 1, "memory": 1.7, "cpu": 2, "utility": 1.7},
|
||||
// {"instance_type": "c1.xlarge", "storage": 1680, "drives": {"$ref": "#4_ephemeral_drives"}, "discount": 0, "ecu": 20, "num_drives": 4, "memory": 7, "cpu": 8, "utility": 7},
|
||||
{"instance_type": "c3.2xlarge", "storage": 160, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 28, "num_drives": 2, "memory": 15, "cpu": 8, "utility": 3.2},
|
||||
{"instance_type": "c3.4xlarge", "storage": 320, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 55, "num_drives": 2, "memory": 30, "cpu": 16, "utility": 6.4},
|
||||
{"instance_type": "c3.8xlarge", "storage": 640, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 108, "num_drives": 2, "memory": 60, "cpu": 32, "utility": 12.8},
|
||||
{"instance_type": "c3.large", "storage": 32, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 7, "num_drives": 2, "memory": 3.75, "cpu": 2, "utility": 0.64},
|
||||
{"instance_type": "c3.xlarge", "storage": 80, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 14, "num_drives": 2, "memory": 7.5, "cpu": 4, "utility": 1.6},
|
||||
// {"instance_type": "cc2.8xlarge", "storage": 3360, "drives": {"$ref": "#4_ephemeral_drives"}, "discount": 0, "ecu": 88, "num_drives": 4, "memory": 60.5, "cpu": 32, "utility": 60},
|
||||
// {"instance_type": "cg1.4xlarge", "storage": 1680, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 33.5, "num_drives": 2, "memory": 22.5, "cpu": 16, "utility": 22.5},
|
||||
{"instance_type": "cr1.8xlarge", "storage": 240, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 88, "num_drives": 2, "memory": 244, "cpu": 32, "utility": 4.8},
|
||||
{"instance_type": "d2.2xlarge", "storage": 12000, "drives": {"$ref": "#6_ephemeral_drives"}, "discount": 0, "ecu": 28, "num_drives": 6, "memory": 61, "cpu": 8, "utility": 60},
|
||||
{"instance_type": "d2.4xlarge", "storage": 24000, "drives": {"$ref": "#8_ephemeral_drives"}, "discount": 0, "ecu": 56, "num_drives": 12, "memory": 122, "cpu": 16, "utility": 60},
|
||||
{"instance_type": "d2.8xlarge", "storage": 48000, "drives": {"$ref": "#8_ephemeral_drives"}, "discount": 0, "ecu": 116, "num_drives": 24, "memory": 244, "cpu": 36, "utility": 60},
|
||||
{"instance_type": "d2.xlarge", "storage": 6000, "drives": {"$ref": "#3_ephemeral_drives"}, "discount": 0, "ecu": 14, "num_drives": 3, "memory": 30.5, "cpu": 4, "utility": 30.5},
|
||||
{"instance_type": "g2.2xlarge", "storage": 60, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 26, "num_drives": 1, "memory": 15, "cpu": 8, "utility": 1.2},
|
||||
{"instance_type": "g2.8xlarge", "storage": 240, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 104, "num_drives": 2, "memory": 60, "cpu": 32, "utility": 4.8},
|
||||
{"instance_type": "hi1.4xlarge", "storage": 2048, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 35, "num_drives": 2, "memory": 60.5, "cpu": 16, "utility": 40.96},
|
||||
// {"instance_type": "hs1.8xlarge", "storage": 48000, "drives": {"$ref": "#8_ephemeral_drives"}, "discount": 0, "ecu": 35, "num_drives": 24, "memory": 117, "cpu": 16, "utility": 60},
|
||||
{"instance_type": "i2.2xlarge", "storage": 1600, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 27, "num_drives": 2, "memory": 61, "cpu": 8, "utility": 32.0},
|
||||
{"instance_type": "i2.4xlarge", "storage": 3200, "drives": {"$ref": "#4_ephemeral_drives"}, "discount": 0, "ecu": 53, "num_drives": 4, "memory": 122, "cpu": 16, "utility": 60},
|
||||
{"instance_type": "i2.8xlarge", "storage": 6400, "drives": {"$ref": "#8_ephemeral_drives"}, "discount": 0, "ecu": 104, "num_drives": 8, "memory": 244, "cpu": 32, "utility": 60},
|
||||
{"instance_type": "i2.xlarge", "storage": 800, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 14, "num_drives": 1, "memory": 30.5, "cpu": 4, "utility": 16.0},
|
||||
// {"instance_type": "i3.16xlarge", "storage": 15200, "drives": {"$ref": "#8_ephemeral_drives"}, "discount": 0, "ecu": 200, "num_drives": 8, "memory": 488, "cpu": 64, "utility": 60},
|
||||
// {"instance_type": "i3.2xlarge", "storage": 1900, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 27, "num_drives": 1, "memory": 61, "cpu": 8, "utility": 38.0},
|
||||
// {"instance_type": "i3.4xlarge", "storage": 3800, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 53, "num_drives": 2, "memory": 122, "cpu": 16, "utility": 60},
|
||||
// {"instance_type": "i3.8xlarge", "storage": 7600, "drives": {"$ref": "#4_ephemeral_drives"}, "discount": 0, "ecu": 99, "num_drives": 4, "memory": 244, "cpu": 32, "utility": 60},
|
||||
// {"instance_type": "i3.large", "storage": 475, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 7, "num_drives": 1, "memory": 15.25, "cpu": 2, "utility": 9.5},
|
||||
// {"instance_type": "i3.xlarge", "storage": 950, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 13, "num_drives": 1, "memory": 30.5, "cpu": 4, "utility": 19.0},
|
||||
// {"instance_type": "m1.large", "storage": 840, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 4, "num_drives": 2, "memory": 7.5, "cpu": 2, "utility": 7.5},
|
||||
// {"instance_type": "m1.medium", "storage": 410, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 2, "num_drives": 1, "memory": 3.75, "cpu": 1, "utility": 3.75},
|
||||
// {"instance_type": "m1.small", "storage": 160, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 1, "num_drives": 1, "memory": 1.7, "cpu": 1, "utility": 1.7},
|
||||
// {"instance_type": "m1.xlarge", "storage": 1680, "drives": {"$ref": "#4_ephemeral_drives"}, "discount": 0, "ecu": 8, "num_drives": 4, "memory": 15, "cpu": 4, "utility": 15},
|
||||
// {"instance_type": "m2.2xlarge", "storage": 850, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 13, "num_drives": 1, "memory": 34.2, "cpu": 4, "utility": 17.0},
|
||||
// {"instance_type": "m2.4xlarge", "storage": 1680, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 26, "num_drives": 2, "memory": 68.4, "cpu": 8, "utility": 33.6},
|
||||
// {"instance_type": "m2.xlarge", "storage": 420, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 6.5, "num_drives": 1, "memory": 17.1, "cpu": 2, "utility": 8.4},
|
||||
{"instance_type": "m3.2xlarge", "storage": 160, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 26, "num_drives": 2, "memory": 30, "cpu": 8, "utility": 3.2},
|
||||
{"instance_type": "m3.large", "storage": 32, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 6.5, "num_drives": 1, "memory": 7.5, "cpu": 2, "utility": 0.64},
|
||||
{"instance_type": "m3.medium", "storage": 4, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 3, "num_drives": 1, "memory": 3.75, "cpu": 1, "utility": 0.08},
|
||||
{"instance_type": "m3.xlarge", "storage": 80, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 13, "num_drives": 2, "memory": 15, "cpu": 4, "utility": 1.6},
|
||||
{"instance_type": "r3.2xlarge", "storage": 160, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 26, "num_drives": 1, "memory": 61, "cpu": 8, "utility": 3.2},
|
||||
{"instance_type": "r3.4xlarge", "storage": 320, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 52, "num_drives": 1, "memory": 122, "cpu": 16, "utility": 6.4},
|
||||
{"instance_type": "r3.8xlarge", "storage": 640, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 104, "num_drives": 2, "memory": 244, "cpu": 32, "utility": 12.8},
|
||||
{"instance_type": "r3.large", "storage": 32, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 6.5, "num_drives": 1, "memory": 15.25, "cpu": 2, "utility": 0.64},
|
||||
{"instance_type": "r3.xlarge", "storage": 80, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 13, "num_drives": 1, "memory": 30.5, "cpu": 4, "utility": 1.6},
|
||||
{"instance_type": "x1.16xlarge", "storage": 1920, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 174.5, "num_drives": 1, "memory": 976, "cpu": 64, "utility": 38.4},
|
||||
{"instance_type": "x1.32xlarge", "storage": 3840, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 349, "num_drives": 2, "memory": 1952, "cpu": 128, "utility": 60}
|
||||
],
|
||||
"ec2": {
|
||||
"request": {
|
||||
//SEE http://boto.readthedocs.org/en/latest/ref/ec2.html#boto.ec2.connection.EC2Connection.request_spot_instances
|
||||
"price": 0.001,
|
||||
"image_id": "ami-e7527ed7",
|
||||
"count": 1,
|
||||
"type": "one-time",
|
||||
"valid_from": null,
|
||||
"expiration": "hour", //SPECIAL, USED TO FILL valid_until
|
||||
"valid_until": null,
|
||||
"launch_group": null,
|
||||
"availability_zone_group": null,
|
||||
"key_name": "activedata",
|
||||
"security_groups": null,
|
||||
"user_data": null,
|
||||
"addressing_type": null,
|
||||
"instance_type": null,
|
||||
"placement": null,
|
||||
"kernel_id": null,
|
||||
"ramdisk_id": null,
|
||||
"monitoring_enabled": false,
|
||||
"subnet_id": null,
|
||||
"placement_group": "es",
|
||||
"block_device_map": null,
|
||||
"instance_profile_arn": null,
|
||||
"instance_profile_name": "active-data",
|
||||
"security_group_ids": null,
|
||||
"ebs_optimized": false,
|
||||
"network_interfaces": {
|
||||
"subnet_id": "subnet-b7c137ee",
|
||||
"groups": ["sg-bb542fde"],
|
||||
"associate_public_ip_address": true
|
||||
},
|
||||
"dry_run": false
|
||||
},
|
||||
"instance": {
|
||||
"name": "ActiveData ES Spot Instance"
|
||||
}
|
||||
},
|
||||
"instance":{
|
||||
"class":"examples.es.ESSpot",
|
||||
"minimum_utility": 2000,
|
||||
"connect": {
|
||||
//USED IN Fabric's `env` GLOBAL CONFIG OBJECT
|
||||
"user": "ec2-user",
|
||||
"key_filename": "~/.ssh/activedata.pem",
|
||||
"disable_known_hosts": true,
|
||||
"host_string": "",
|
||||
"port": 22,
|
||||
"password": "",
|
||||
"banner_timeout": 30
|
||||
},
|
||||
"new_volume":{
|
||||
"size":1000,
|
||||
"volume_type":"magnetic",
|
||||
"zone": "us-west-2c",
|
||||
"snapshot": null,
|
||||
"iops":null,
|
||||
"encrypted":false
|
||||
}
|
||||
},
|
||||
"debug": {
|
||||
"trace": true,
|
||||
"cprofile": {
|
||||
"enabled": false,
|
||||
"filename": "results/examples_spot_profile.tab"
|
||||
},
|
||||
"log": [
|
||||
{
|
||||
"class": "logging.handlers.RotatingFileHandler",
|
||||
"filename": "examples/logs/examples_es.log",
|
||||
"maxBytes": 10000000,
|
||||
"backupCount": 10,
|
||||
"encoding": "utf8"
|
||||
},
|
||||
{
|
||||
"log_type": "ses",
|
||||
"from_address": "klahnakoski@mozilla.com",
|
||||
"to_address": "klahnakoski@mozilla.com",
|
||||
"subject": "[ALERT][Manager] Problem in ES Spot",
|
||||
"$ref": "file://~/private.json#aws_credentials"
|
||||
},
|
||||
{
|
||||
"log_type": "console"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
@ -1,44 +0,0 @@
|
|||
# PUT THIS FILE IN /etc/supervisord.conf
|
||||
|
||||
[supervisord]
|
||||
logfile=/data1/logs/supervisord.log
|
||||
logfile_maxbytes=50MB
|
||||
logfile_backups=10
|
||||
minfds=100000
|
||||
|
||||
[unix_http_server]
|
||||
file=/etc/supervisor.sock ; (the path to the socket file)
|
||||
|
||||
[rpcinterface:supervisor]
|
||||
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
|
||||
|
||||
[supervisorctl]
|
||||
serverurl=unix:///etc/supervisor.sock
|
||||
|
||||
[program:es]
|
||||
command=/usr/local/elasticsearch/bin/elasticsearch
|
||||
directory=/usr/local/elasticsearch
|
||||
autostart=true
|
||||
autorestart=true
|
||||
startretries=10
|
||||
stopsignal=INT
|
||||
stopwaitsecs=600
|
||||
stderr_logfile=/data1/logs/es.error.log
|
||||
stdout_logfile=/data1/logs/es.log
|
||||
stdout_logfile_maxbytes=10MB
|
||||
stdout_logfile_backups=10
|
||||
user=root
|
||||
environment=JAVA_HOME=/usr/java/default
|
||||
|
||||
[program:push_to_es]
|
||||
command=python27 activedata_etl/push_to_es.py --settings=resources/settings/staging/push_to_es.json
|
||||
directory=/home/ec2-user/ActiveData-ETL
|
||||
autostart=true
|
||||
autorestart=true
|
||||
stopsignal=INT
|
||||
stopwaitsecs=30
|
||||
stderr_logfile=/data1/logs/push_to_es.error.log
|
||||
stdout_logfile=/data1/logs/push_to_es.log
|
||||
user=ec2-user
|
||||
environment=PYTHONPATH=.;HOME=/home/ec2-user
|
||||
|
||||
|
|
@ -41,13 +41,13 @@
|
|||
{"instance_type": "h1.16xlarge", "num_drives": 24, "memory": 256, "cpu": 64, "utility": 64},
|
||||
|
||||
// {"instance_type": "m1.small", "ecu": 3, "num_drives": 1, "memory": 1.7, "cpu": 1, "utility": 1}, # TOO SMALL
|
||||
{"instance_type": "m1.medium", "ecu": 3, "num_drives": 1, "memory": 3.75, "cpu": 1, "utility": 1},
|
||||
{"instance_type": "m1.large", "ecu": 6.5, "num_drives": 2, "memory": 7.5, "cpu": 2, "utility": 2},
|
||||
{"instance_type": "m1.xlarge", "ecu": 13, "num_drives": 4, "memory": 15, "cpu": 4, "utility": 4},
|
||||
|
||||
{"instance_type": "m2.xlarge", "num_drives": 1, "memory": 17.1, "cpu": 2, "utility": 2},
|
||||
{"instance_type": "m2.2xlarge", "num_drives": 1, "memory": 34, "cpu": 4, "utility": 4},
|
||||
{"instance_type": "m2.4xlarge", "num_drives": 2, "memory": 68, "cpu": 8, "utility": 8},
|
||||
// {"instance_type": "m1.medium", "ecu": 3, "num_drives": 1, "memory": 3.75, "cpu": 1, "utility": 1},
|
||||
// {"instance_type": "m1.large", "ecu": 6.5, "num_drives": 2, "memory": 7.5, "cpu": 2, "utility": 2},
|
||||
// {"instance_type": "m1.xlarge", "ecu": 13, "num_drives": 4, "memory": 15, "cpu": 4, "utility": 4},
|
||||
//
|
||||
// {"instance_type": "m2.xlarge", "num_drives": 1, "memory": 17.1, "cpu": 2, "utility": 2},
|
||||
// {"instance_type": "m2.2xlarge", "num_drives": 1, "memory": 34, "cpu": 4, "utility": 4},
|
||||
// {"instance_type": "m2.4xlarge", "num_drives": 2, "memory": 68, "cpu": 8, "utility": 8},
|
||||
|
||||
{"instance_type": "m3.medium", "num_drives": 1, "memory": 3.75, "cpu": 1, "utility": 1},
|
||||
{"instance_type": "m3.large", "num_drives": 1, "memory": 7.5, "cpu": 2, "utility": 2},
|
||||
|
|
@ -127,7 +127,7 @@
|
|||
}
|
||||
},
|
||||
"constants":{
|
||||
"pyLibrary.env.http.default_headers":{"Referer": "https://github.com/klahnakoski/SpotManager"}
|
||||
"mo_http.http.default_headers":{"Referer": "https://github.com/klahnakoski/SpotManager"}
|
||||
},
|
||||
"debug": {
|
||||
"trace": true,
|
||||
|
|
|
|||
|
|
@ -25,6 +25,6 @@
|
|||
},
|
||||
"utility": {"cpu": 1, "utility":1},
|
||||
"constants":{
|
||||
"pyLibrary.env.http.default_headers.referer":"https://wiki.mozilla.org/Auto-tools/Projects/ActiveData"
|
||||
"mo_http.http.default_headers.referer":"https://wiki.mozilla.org/Auto-tools/Projects/ActiveData"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
235
examples/es.py
235
examples/es.py
|
|
@ -1,235 +0,0 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Author: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
from __future__ import division
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from mo_fabric import Connection
|
||||
from mo_files import File
|
||||
from mo_future import text_type
|
||||
from mo_kwargs import override
|
||||
from mo_logs import Log
|
||||
from mo_logs.strings import expand_template
|
||||
from mo_math import Math
|
||||
from mo_math.randoms import Random
|
||||
from spot.instance_manager import InstanceManager
|
||||
|
||||
JRE = "jre-8u131-linux-x64.rpm"
|
||||
LOCAL_JRE = "resources/" + JRE
|
||||
|
||||
|
||||
class ESSpot(InstanceManager):
|
||||
"""
|
||||
THIS CLASS MUST HAVE AN IMPLEMENTATION FOR the SpotManager TO USE
|
||||
"""
|
||||
@override
|
||||
def __init__(self, minimum_utility, kwargs=None):
|
||||
InstanceManager.__init__(self, kwargs)
|
||||
self.settings = kwargs
|
||||
self.minimum_utility = minimum_utility
|
||||
|
||||
def required_utility(self, current_utility=None):
|
||||
return self.minimum_utility
|
||||
|
||||
def setup(
|
||||
self,
|
||||
instance, # THE boto INSTANCE OBJECT FOR THE MACHINE TO SETUP
|
||||
utility, # THE utility OBJECT FOUND IN CONFIG
|
||||
please_stop
|
||||
):
|
||||
with Connection(host=instance.ip_address, kwargs=self.settings.connect) as conn:
|
||||
gigabytes = Math.floor(utility.memory)
|
||||
Log.note("setup {{instance}}", instance=instance.id)
|
||||
|
||||
self._install_indexer(instance, conn)
|
||||
self._install_es(gigabytes, instance, conn)
|
||||
self._install_supervisor(instance, conn)
|
||||
self._start_supervisor(conn)
|
||||
|
||||
def teardown(
|
||||
self,
|
||||
instance, # THE boto INSTANCE OBJECT FOR THE MACHINE TO TEARDOWN
|
||||
please_stop
|
||||
):
|
||||
with Connection(host=instance.ip_address, kwargs=self.settings.connect) as conn:
|
||||
Log.note("teardown {{instance}}", instance=instance.id)
|
||||
|
||||
# ASK NICELY TO STOP Elasticsearch PROCESS
|
||||
conn.sudo("supervisorctl stop es", warn=True)
|
||||
|
||||
# ASK NICELY TO STOP "supervisord" PROCESS
|
||||
conn.sudo("ps -ef | grep supervisord | grep -v grep | awk '{print $2}' | xargs kill -SIGINT", warn=True)
|
||||
|
||||
# WAIT FOR SUPERVISOR SHUTDOWN
|
||||
pid = True
|
||||
while pid:
|
||||
pid = conn.sudo("ps -ef | grep supervisord | grep -v grep | awk '{print $2}'")
|
||||
|
||||
def _install_es(self, gigabytes, instance, conn):
|
||||
volumes = instance.markup.drives
|
||||
|
||||
if not conn.exists("/usr/local/elasticsearch"):
|
||||
with conn.cd("/home/ec2-user/"):
|
||||
conn.run("mkdir -p temp")
|
||||
|
||||
if not File(LOCAL_JRE).exists:
|
||||
Log.error("Expecting {{file}} on manager to spread to ES instances", file=LOCAL_JRE)
|
||||
with conn.cd("/home/ec2-user/temp"):
|
||||
conn.run('rm -f '+JRE)
|
||||
conn.put("resources/"+JRE, JRE)
|
||||
conn.sudo("rpm -i "+JRE)
|
||||
conn.sudo("alternatives --install /usr/bin/java java /usr/java/default/bin/java 20000")
|
||||
conn.run("export JAVA_HOME=/usr/java/default")
|
||||
|
||||
with conn.cd("/home/ec2-user/"):
|
||||
conn.run('wget https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.7.1.tar.gz')
|
||||
conn.run('tar zxfv elasticsearch-1.7.1.tar.gz')
|
||||
conn.sudo('mkdir /usr/local/elasticsearch')
|
||||
conn.sudo('cp -R elasticsearch-1.7.1/* /usr/local/elasticsearch/')
|
||||
|
||||
with conn.cd('/usr/local/elasticsearch/'):
|
||||
# BE SURE TO MATCH THE PLUGLIN WITH ES VERSION
|
||||
# https://github.com/elasticsearch/elasticsearch-cloud-aws
|
||||
conn.sudo('bin/plugin -install elasticsearch/elasticsearch-cloud-aws/2.7.1')
|
||||
|
||||
# REMOVE THESE FILES, WE WILL REPLACE THEM WITH THE CORRECT VERSIONS AT THE END
|
||||
conn.sudo("rm -f /usr/local/elasticsearch/config/elasticsearch.yml")
|
||||
conn.sudo("rm -f /usr/local/elasticsearch/bin/elasticsearch.in.sh")
|
||||
|
||||
# MOUNT AND FORMAT THE EBS VOLUMES (list with `lsblk`)
|
||||
for i, k in enumerate(volumes):
|
||||
if not conn.exists(k.path):
|
||||
conn.sudo('sudo umount '+k.device, warn=True)
|
||||
|
||||
conn.sudo('yes | sudo mkfs -t ext4 '+k.device)
|
||||
conn.sudo('mkdir '+k.path)
|
||||
conn.sudo('sudo mount '+k.device+' '+k.path)
|
||||
|
||||
# ADD TO /etc/fstab SO AROUND AFTER REBOOT
|
||||
conn.sudo("sed -i '$ a\\"+k.device+" "+k.path+" ext4 defaults,nofail 0 2' /etc/fstab")
|
||||
|
||||
# TEST IT IS WORKING
|
||||
conn.sudo('mount -a')
|
||||
|
||||
# INCREASE THE FILE HANDLE LIMITS
|
||||
with conn.cd("/home/ec2-user/"):
|
||||
File("./results/temp/sysctl.conf").delete()
|
||||
conn.get("/etc/sysctl.conf", "./results/temp/sysctl.conf", use_sudo=True)
|
||||
lines = File("./results/temp/sysctl.conf").read()
|
||||
if lines.find("fs.file-max = 100000") == -1:
|
||||
lines += "\nfs.file-max = 100000"
|
||||
lines = lines.replace("net.bridge.bridge-nf-call-ip6tables = 0", "")
|
||||
lines = lines.replace("net.bridge.bridge-nf-call-iptables = 0", "")
|
||||
lines = lines.replace("net.bridge.bridge-nf-call-arptables = 0", "")
|
||||
File("./results/temp/sysctl.conf").write(lines)
|
||||
conn.put("./results/temp/sysctl.conf", "/etc/sysctl.conf", use_sudo=True)
|
||||
|
||||
conn.sudo("sysctl -p")
|
||||
|
||||
# INCREASE FILE HANDLE PERMISSIONS
|
||||
conn.sudo("sed -i '$ a\\root soft nofile 100000' /etc/security/limits.conf")
|
||||
conn.sudo("sed -i '$ a\\root hard nofile 100000' /etc/security/limits.conf")
|
||||
conn.sudo("sed -i '$ a\\root memlock unlimited' /etc/security/limits.conf")
|
||||
|
||||
conn.sudo("sed -i '$ a\\ec2-user soft nofile 100000' /etc/security/limits.conf")
|
||||
conn.sudo("sed -i '$ a\\ec2-user hard nofile 100000' /etc/security/limits.conf")
|
||||
conn.sudo("sed -i '$ a\\ec2-user memlock unlimited' /etc/security/limits.conf")
|
||||
|
||||
# EFFECTIVE LOGIN TO LOAD CHANGES TO FILE HANDLES
|
||||
# conn.sudo("sudo -i -u ec2-user")
|
||||
|
||||
if not conn.exists("/data1/logs"):
|
||||
conn.sudo('mkdir /data1/logs')
|
||||
conn.sudo('mkdir /data1/heapdump')
|
||||
|
||||
# INCREASE NUMBER OF FILE HANDLES
|
||||
# conn.sudo("sysctl -w fs.file-max=64000")
|
||||
# COPY CONFIG FILE TO ES DIR
|
||||
if not conn.exists("/usr/local/elasticsearch/config/elasticsearch.yml"):
|
||||
yml = File("./examples/config/es_config.yml").read().replace("\r", "")
|
||||
yml = expand_template(yml, {
|
||||
"id": Random.hex(length=8),
|
||||
"data_paths": ",".join("/data"+text_type(i+1) for i, _ in enumerate(volumes))
|
||||
})
|
||||
File("./results/temp/elasticsearch.yml").write(yml)
|
||||
conn.put("./results/temp/elasticsearch.yml", '/usr/local/elasticsearch/config/elasticsearch.yml', use_sudo=True)
|
||||
|
||||
# FOR SOME REASON THE export COMMAND DOES NOT SEEM TO WORK
|
||||
# THIS SCRIPT SETS THE ES_MIN_MEM/ES_MAX_MEM EXPLICITLY
|
||||
if not conn.exists("/usr/local/elasticsearch/bin/elasticsearch.in.sh"):
|
||||
sh = File("./examples/config/es_run.sh").read().replace("\r", "")
|
||||
sh = expand_template(sh, {"memory": text_type(int(gigabytes/2))})
|
||||
File("./results/temp/elasticsearch.in.sh").write(sh)
|
||||
with conn.cd("/home/ec2-user"):
|
||||
conn.put("./results/temp/elasticsearch.in.sh", './temp/elasticsearch.in.sh', use_sudo=True)
|
||||
conn.sudo("cp -f ./temp/elasticsearch.in.sh /usr/local/elasticsearch/bin/elasticsearch.in.sh")
|
||||
|
||||
def _install_indexer(self, instance, conn):
|
||||
Log.note("Install indexer at {{instance_id}} ({{address}})", instance_id=instance.id, address=instance.ip_address)
|
||||
self._install_python(instance, conn)
|
||||
|
||||
if not conn.exists("/home/ec2-user/ActiveData-ETL/"):
|
||||
with conn.cd("/home/ec2-user"):
|
||||
conn.sudo("yum -y install git")
|
||||
conn.run("git clone https://github.com/klahnakoski/ActiveData-ETL.git")
|
||||
|
||||
with conn.cd("/home/ec2-user/ActiveData-ETL/"):
|
||||
conn.run("git checkout push-to-es")
|
||||
conn.sudo("yum -y install gcc") # REQUIRED FOR psutil
|
||||
conn.sudo("pip install -r requirements.txt")
|
||||
|
||||
conn.put("~/private_active_data_etl.json", "/home/ec2-user/private.json")
|
||||
|
||||
def _install_python(self, instance, conn):
|
||||
Log.note("Install Python at {{instance_id}} ({{address}})", instance_id=instance.id, address=instance.ip_address)
|
||||
if conn.exists("/usr/bin/pip"):
|
||||
pip_version = conn.sudo("pip --version", warn=True)
|
||||
else:
|
||||
pip_version = ""
|
||||
|
||||
if not pip_version.startswith("pip 9."):
|
||||
conn.sudo("yum -y install python27")
|
||||
conn.sudo("easy_install pip")
|
||||
conn.sudo("rm -f /usr/bin/pip", warn=True)
|
||||
conn.sudo("ln -s /usr/local/bin/pip /usr/bin/pip")
|
||||
conn.sudo("pip install --upgrade pip")
|
||||
|
||||
def _install_supervisor(self, instance, conn):
|
||||
Log.note("Install Supervisor-plus-Cron at {{instance_id}} ({{address}})", instance_id=instance.id, address=instance.ip_address)
|
||||
# REQUIRED FOR Python SSH
|
||||
self._install_lib("libffi-devel", conn)
|
||||
self._install_lib("openssl-devel", conn)
|
||||
self._install_lib('"Development tools"', install="groupinstall", conn=conn)
|
||||
|
||||
self._install_python(instance, conn)
|
||||
conn.sudo("pip install pyopenssl")
|
||||
conn.sudo("pip install ndg-httpsclient")
|
||||
conn.sudo("pip install pyasn1")
|
||||
conn.sudo("pip install fabric==1.10.2")
|
||||
conn.sudo("pip install requests")
|
||||
|
||||
conn.sudo("pip install supervisor-plus-cron")
|
||||
|
||||
def _install_lib(self, lib_name, install="install", conn=None):
|
||||
"""
|
||||
:param lib_name:
|
||||
:param install: use 'groupinstall' if you wish
|
||||
:return:
|
||||
"""
|
||||
result = conn.sudo("yum "+install+" -y "+lib_name, warn=True)
|
||||
if result.return_code != 0 and "already installed and latest version" not in result:
|
||||
Log.error("problem with install of {{lib}}", lib=lib_name)
|
||||
|
||||
def _start_supervisor(self, conn):
|
||||
conn.put("./examples/config/es_supervisor.conf", "/etc/supervisord.conf", use_sudo=True)
|
||||
|
||||
# START DAEMON (OR THROW ERROR IF RUNNING ALREADY)
|
||||
conn.sudo("supervisord -c /etc/supervisord.conf", warn=True)
|
||||
|
||||
conn.sudo("supervisorctl reread")
|
||||
conn.sudo("supervisorctl update")
|
||||
380
examples/es6.py
380
examples/es6.py
|
|
@ -10,20 +10,18 @@ from __future__ import division
|
|||
from __future__ import unicode_literals
|
||||
|
||||
from mo_fabric import Connection
|
||||
from mo_files import File
|
||||
from mo_future import text_type
|
||||
from mo_files import File, TempFile
|
||||
from mo_future import text
|
||||
from mo_kwargs import override
|
||||
from mo_logs import Log
|
||||
from mo_logs.strings import expand_template
|
||||
from mo_math import Math
|
||||
import mo_math
|
||||
from spot.instance_manager import InstanceManager
|
||||
|
||||
JRE = "jre-8u131-linux-x64.rpm"
|
||||
LOCAL_JRE = "resources/" + JRE
|
||||
|
||||
PYPY_DIR = "pypy-6.0.0-linux_x86_64-portable"
|
||||
PYPY_BZ2 = "pypy-6.0.0-linux_x86_64-portable.tar.bz2"
|
||||
LOCAL_PYPY = "resources/" + PYPY_BZ2
|
||||
PYPY_DIR = "pypy2.7-v7.3.0-linux64"
|
||||
PYPY_BZ2 = "pypy2.7-v7.3.0-linux64.tar.bz2"
|
||||
RESOURCES = File("resources")
|
||||
|
||||
|
||||
class ES6Spot(InstanceManager):
|
||||
|
|
@ -45,14 +43,18 @@ class ES6Spot(InstanceManager):
|
|||
utility, # THE utility OBJECT FOUND IN CONFIG
|
||||
please_stop
|
||||
):
|
||||
with Connection(host=instance.ip_address, kwargs=self.settings.connect) as conn:
|
||||
gigabytes = Math.floor(utility.memory)
|
||||
Log.note("setup {{instance}}", instance=instance.id)
|
||||
try:
|
||||
with Connection(host=instance.ip_address, kwargs=self.settings.connect) as conn:
|
||||
gigabytes = mo_math.floor(utility.memory)
|
||||
Log.note("setup {{instance}}", instance=instance.id)
|
||||
|
||||
self._install_pypy_indexer(instance, conn)
|
||||
self._install_es(gigabytes, instance, conn)
|
||||
self._install_supervisor(instance, conn)
|
||||
self._start_supervisor(conn)
|
||||
_install_pypy_indexer(instance=instance, conn=conn)
|
||||
_install_es(gigabytes, instance=instance, conn=conn)
|
||||
_install_supervisor(instance=instance, conn=conn)
|
||||
_start_supervisor(conn=conn)
|
||||
Log.alert("Done install of {{host}}", host=instance.ip_address)
|
||||
except Exception as e:
|
||||
Log.error("could not setup ES at {{ip}}", ip=instance.ip_address, cause=e)
|
||||
|
||||
def teardown(
|
||||
self,
|
||||
|
|
@ -62,223 +64,221 @@ class ES6Spot(InstanceManager):
|
|||
with Connection(host=instance.ip_address, kwargs=self.settings.connect) as conn:
|
||||
Log.note("teardown {{instance}}", instance=instance.id)
|
||||
|
||||
# ASK NICELY TO STOP Elasticsearch PROCESS
|
||||
conn.sudo("supervisorctl stop push-to-es:*", warn=True)
|
||||
# ASK NICELY TO STOP Elasticsearch PROCESS
|
||||
conn.sudo("supervisorctl stop es:*", warn=True)
|
||||
|
||||
# ASK NICELY TO STOP "supervisord" PROCESS
|
||||
conn.sudo("ps -ef | grep supervisord | grep -v grep | awk '{print $2}' | xargs kill -SIGINT", warn=True)
|
||||
pid = conn.sudo("ps -ef | grep supervisord | grep -v grep | awk '{print $2}'", warn=True).stdout.strip()
|
||||
Log.note("shutdown supervisor at pid={{pid}}", pid=pid)
|
||||
conn.sudo("kill -SIGINT " + pid, warn=True)
|
||||
|
||||
# WAIT FOR SUPERVISOR SHUTDOWN
|
||||
pid = True
|
||||
while pid:
|
||||
pid = conn.sudo("ps -ef | grep supervisord | grep -v grep | awk '{print $2}'")
|
||||
pid = conn.sudo("ps -ef | grep supervisord | grep -v grep | awk '{print $2}'").stdout.strip()
|
||||
|
||||
def _install_es(gigabytes, es_version="6.5.4", instance=None, conn=None):
|
||||
es_file = 'elasticsearch-' + es_version + '.tar.gz'
|
||||
volumes = instance.markup.drives
|
||||
|
||||
def _install_es(self, gigabytes, es_version="6.2.3", instance=None, conn=None):
|
||||
volumes = instance.markup.drives
|
||||
|
||||
if not conn.exists("/usr/local/elasticsearch/config/elasticsearch.yml"):
|
||||
with conn.cd("/home/ec2-user/"):
|
||||
conn.run("mkdir -p temp")
|
||||
|
||||
if not File(LOCAL_JRE).exists:
|
||||
Log.error("Expecting {{file}} on manager to spread to ES instances", file=LOCAL_JRE)
|
||||
response = conn.run("java -version", warn=True)
|
||||
if "Java(TM) SE Runtime Environment" not in response:
|
||||
with conn.cd("/home/ec2-user/temp"):
|
||||
conn.run('rm -f '+JRE)
|
||||
conn.put(LOCAL_JRE, JRE)
|
||||
conn.sudo("rpm -i "+JRE)
|
||||
conn.sudo("alternatives --install /usr/bin/java java /usr/java/default/bin/java 20000")
|
||||
conn.run("export JAVA_HOME=/usr/java/default")
|
||||
|
||||
with conn.cd("/home/ec2-user/"):
|
||||
conn.run('wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-'+es_version+'.tar.gz')
|
||||
conn.run('tar zxfv elasticsearch-'+es_version+'.tar.gz')
|
||||
conn.sudo("rm -fr /usr/local/elasticsearch", warn=True)
|
||||
conn.sudo('mkdir /usr/local/elasticsearch')
|
||||
conn.sudo('cp -R elasticsearch-'+es_version+'/* /usr/local/elasticsearch/')
|
||||
|
||||
with conn.cd('/usr/local/elasticsearch/'):
|
||||
# BE SURE TO MATCH THE PLUGLIN WITH ES VERSION
|
||||
# https://github.com/elasticsearch/elasticsearch-cloud-aws
|
||||
conn.sudo('sudo bin/elasticsearch-plugin install -b discovery-ec2')
|
||||
|
||||
# REMOVE THESE FILES, WE WILL REPLACE THEM WITH THE CORRECT VERSIONS AT THE END
|
||||
conn.sudo("rm -f /usr/local/elasticsearch/config/elasticsearch.yml")
|
||||
conn.sudo("rm -f /usr/local/elasticsearch/config/jvm.options")
|
||||
conn.sudo("rm -f /usr/local/elasticsearch/config/log4j2.properties")
|
||||
|
||||
self.conn = instance.connection
|
||||
|
||||
# MOUNT AND FORMAT THE VOLUMES (list with `lsblk`)
|
||||
for i, k in enumerate(volumes):
|
||||
if not conn.exists(k.path):
|
||||
conn.sudo('sudo umount '+k.device, warn=True)
|
||||
|
||||
conn.sudo('yes | sudo mkfs -t ext4 '+k.device)
|
||||
|
||||
# ES AND JOURNALLING DO NOT MIX
|
||||
conn.sudo('tune2fs -o journal_data_writeback '+k.device)
|
||||
conn.sudo('tune2fs -O ^has_journal '+k.device)
|
||||
conn.sudo('mkdir '+k.path)
|
||||
conn.sudo('sudo mount '+k.device+' '+k.path)
|
||||
conn.sudo('chown -R ec2-user:ec2-user '+k.path)
|
||||
|
||||
# ADD TO /etc/fstab SO AROUND AFTER REBOOT
|
||||
conn.sudo("sed -i '$ a\\"+k.device+" "+k.path+" ext4 defaults,nofail 0 2' /etc/fstab")
|
||||
|
||||
# TEST IT IS WORKING
|
||||
conn.sudo('mount -a')
|
||||
|
||||
# INCREASE THE FILE HANDLE LIMITS
|
||||
if not conn.exists("/usr/local/elasticsearch/config/elasticsearch.yml"):
|
||||
with conn.cd("/home/ec2-user/"):
|
||||
File("./results/temp/sysctl.conf").delete()
|
||||
conn.get("/etc/sysctl.conf", "./results/temp/sysctl.conf", use_sudo=True)
|
||||
lines = File("./results/temp/sysctl.conf").read()
|
||||
conn.run("mkdir -p temp")
|
||||
|
||||
if not (RESOURCES / JRE).exists:
|
||||
Log.error("Expecting {{file}} on manager to spread to ES instances", file=(RESOURCES / JRE))
|
||||
response = conn.run("java -version", warn=True)
|
||||
if "Java(TM) SE Runtime Environment" not in response:
|
||||
with conn.cd("/home/ec2-user/temp"):
|
||||
conn.run('rm -f '+JRE)
|
||||
conn.put((RESOURCES / JRE), JRE)
|
||||
conn.sudo("rpm -i "+JRE)
|
||||
conn.sudo("alternatives --install /usr/bin/java java /usr/java/default/bin/java 20000")
|
||||
conn.run("export JAVA_HOME=/usr/java/default")
|
||||
|
||||
with conn.cd("/home/ec2-user/"):
|
||||
conn.put(RESOURCES / es_file, es_file)
|
||||
conn.run('tar zxfv ' + es_file)
|
||||
conn.sudo("rm -fr /usr/local/elasticsearch", warn=True)
|
||||
conn.sudo('mkdir /usr/local/elasticsearch')
|
||||
conn.sudo('cp -R elasticsearch-'+es_version+'/* /usr/local/elasticsearch/')
|
||||
|
||||
with conn.cd('/usr/local/elasticsearch/'):
|
||||
# BE SURE TO MATCH THE PLUGLIN WITH ES VERSION
|
||||
# https://github.com/elasticsearch/elasticsearch-cloud-aws
|
||||
conn.sudo('sudo bin/elasticsearch-plugin install -b discovery-ec2')
|
||||
|
||||
# REMOVE THESE FILES, WE WILL REPLACE THEM WITH THE CORRECT VERSIONS AT THE END
|
||||
conn.sudo("rm -f /usr/local/elasticsearch/config/elasticsearch.yml")
|
||||
conn.sudo("rm -f /usr/local/elasticsearch/config/jvm.options")
|
||||
conn.sudo("rm -f /usr/local/elasticsearch/config/log4j2.properties")
|
||||
|
||||
# MOUNT AND FORMAT THE VOLUMES (list with `lsblk`)
|
||||
for i, k in enumerate(volumes):
|
||||
if not conn.exists(k.path):
|
||||
# ENSURE DEVICE IS NOT MOUNTED
|
||||
conn.sudo('sudo umount '+k.device, warn=True)
|
||||
|
||||
# (RE)PARTITION THE LOCAL DEVICE, AND FORMAT
|
||||
conn.sudo("parted " + k.device + " --script \"mklabel gpt mkpart primary ext4 2048s 100%\"")
|
||||
conn.sudo('yes | sudo mkfs -t ext4 '+k.device)
|
||||
|
||||
# ES AND JOURNALLING DO NOT MIX
|
||||
conn.sudo('tune2fs -o journal_data_writeback '+k.device)
|
||||
conn.sudo('tune2fs -O ^has_journal '+k.device)
|
||||
|
||||
# MOUNT IT
|
||||
conn.sudo('mkdir '+k.path)
|
||||
conn.sudo('sudo mount '+k.device+' '+k.path)
|
||||
conn.sudo('chown -R ec2-user:ec2-user '+k.path)
|
||||
|
||||
# ADD TO /etc/fstab SO AROUND AFTER REBOOT
|
||||
conn.sudo("sed -i '$ a\\"+k.device+" "+k.path+" ext4 defaults,nofail 0 2' /etc/fstab")
|
||||
|
||||
# TEST IT IS WORKING
|
||||
conn.sudo('mount -a')
|
||||
|
||||
# INCREASE THE FILE HANDLE LIMITS
|
||||
with conn.cd("/home/ec2-user/"):
|
||||
with TempFile() as temp:
|
||||
conn.get("/etc/sysctl.conf", temp, use_sudo=True)
|
||||
lines = temp.read()
|
||||
if lines.find("fs.file-max = 100000") == -1:
|
||||
lines += "\nfs.file-max = 100000"
|
||||
lines = lines.replace("net.bridge.bridge-nf-call-ip6tables = 0", "")
|
||||
lines = lines.replace("net.bridge.bridge-nf-call-iptables = 0", "")
|
||||
lines = lines.replace("net.bridge.bridge-nf-call-arptables = 0", "")
|
||||
File("./results/temp/sysctl.conf").write(lines)
|
||||
conn.put("./results/temp/sysctl.conf", "/etc/sysctl.conf", use_sudo=True)
|
||||
temp.write(lines)
|
||||
conn.put(temp, "/etc/sysctl.conf", use_sudo=True)
|
||||
|
||||
conn.sudo("sudo sed -i '$ a\\vm.max_map_count = 262144' /etc/sysctl.conf")
|
||||
conn.sudo("sudo sed -i '$ a\\vm.max_map_count = 262144' /etc/sysctl.conf")
|
||||
|
||||
conn.sudo("sysctl -p")
|
||||
conn.sudo("sysctl -p")
|
||||
|
||||
# INCREASE FILE HANDLE PERMISSIONS
|
||||
conn.sudo("sed -i '$ a\\root soft nofile 100000' /etc/security/limits.conf")
|
||||
conn.sudo("sed -i '$ a\\root hard nofile 100000' /etc/security/limits.conf")
|
||||
conn.sudo("sed -i '$ a\\root soft memlock unlimited' /etc/security/limits.conf")
|
||||
conn.sudo("sed -i '$ a\\root hard memlock unlimited' /etc/security/limits.conf")
|
||||
# INCREASE FILE HANDLE PERMISSIONS
|
||||
conn.sudo("sed -i '$ a\\root soft nofile 100000' /etc/security/limits.conf")
|
||||
conn.sudo("sed -i '$ a\\root hard nofile 100000' /etc/security/limits.conf")
|
||||
conn.sudo("sed -i '$ a\\root soft memlock unlimited' /etc/security/limits.conf")
|
||||
conn.sudo("sed -i '$ a\\root hard memlock unlimited' /etc/security/limits.conf")
|
||||
|
||||
conn.sudo("sed -i '$ a\\ec2-user soft nofile 100000' /etc/security/limits.conf")
|
||||
conn.sudo("sed -i '$ a\\ec2-user hard nofile 100000' /etc/security/limits.conf")
|
||||
conn.sudo("sed -i '$ a\\ec2-user soft memlock unlimited' /etc/security/limits.conf")
|
||||
conn.sudo("sed -i '$ a\\ec2-user hard memlock unlimited' /etc/security/limits.conf")
|
||||
conn.sudo("sed -i '$ a\\ec2-user soft nofile 100000' /etc/security/limits.conf")
|
||||
conn.sudo("sed -i '$ a\\ec2-user hard nofile 100000' /etc/security/limits.conf")
|
||||
conn.sudo("sed -i '$ a\\ec2-user soft memlock unlimited' /etc/security/limits.conf")
|
||||
conn.sudo("sed -i '$ a\\ec2-user hard memlock unlimited' /etc/security/limits.conf")
|
||||
|
||||
if not conn.exists("/data1/logs"):
|
||||
conn.run('mkdir /data1/logs')
|
||||
conn.run('mkdir /data1/heapdump')
|
||||
|
||||
if not conn.exists("/data1/logs"):
|
||||
conn.run('mkdir /data1/logs')
|
||||
conn.run('mkdir /data1/heapdump')
|
||||
# COPY CONFIG FILES TO ES DIR
|
||||
if not conn.exists("/usr/local/elasticsearch/config/elasticsearch.yml"):
|
||||
conn.put("./examples/config/es6_log4j2.properties", '/usr/local/elasticsearch/config/log4j2.properties', use_sudo=True)
|
||||
|
||||
# COPY CONFIG FILES TO ES DIR
|
||||
if not conn.exists("/usr/local/elasticsearch/config/elasticsearch.yml"):
|
||||
conn.put("./examples/config/es6_log4j2.properties", '/usr/local/elasticsearch/config/log4j2.properties', use_sudo=True)
|
||||
jvm = File("./examples/config/es6_jvm.options").read().replace('\r', '')
|
||||
jvm = expand_template(jvm, {"memory": int(gigabytes/2)})
|
||||
with TempFile() as temp:
|
||||
temp.write(jvm)
|
||||
conn.put(temp, '/usr/local/elasticsearch/config/jvm.options', use_sudo=True)
|
||||
|
||||
jvm = File("./examples/config/es6_jvm.options").read().replace('\r', '')
|
||||
jvm = expand_template(jvm, {"memory": int(gigabytes/2)})
|
||||
File("./results/temp/jvm.options").write(jvm)
|
||||
conn.put("./results/temp/jvm.options", '/usr/local/elasticsearch/config/jvm.options', use_sudo=True)
|
||||
yml = File("./examples/config/es6_config.yml").read().replace("\r", "")
|
||||
yml = expand_template(yml, {
|
||||
"id": instance.ip_address,
|
||||
"data_paths": ",".join("/data" + text(i + 1) for i, _ in enumerate(volumes))
|
||||
})
|
||||
with TempFile() as temp:
|
||||
temp.write(yml)
|
||||
conn.put(temp, '/usr/local/elasticsearch/config/elasticsearch.yml', use_sudo=True)
|
||||
|
||||
yml = File("./examples/config/es6_config.yml").read().replace("\r", "")
|
||||
yml = expand_template(yml, {
|
||||
"id": instance.ip_address,
|
||||
"data_paths": ",".join("/data" + text_type(i + 1) for i, _ in enumerate(volumes))
|
||||
})
|
||||
File("./results/temp/elasticsearch.yml").write(yml)
|
||||
conn.put("./results/temp/elasticsearch.yml", '/usr/local/elasticsearch/config/elasticsearch.yml', use_sudo=True)
|
||||
conn.sudo("chown -R ec2-user:ec2-user /usr/local/elasticsearch")
|
||||
|
||||
conn.sudo("chown -R ec2-user:ec2-user /usr/local/elasticsearch")
|
||||
def _install_python_indexer(instance, conn):
|
||||
Log.note("Install Python at {{instance_id}} ({{address}})", instance_id=instance.id, address=instance.ip_address)
|
||||
_install_python(instance, conn)
|
||||
|
||||
def _install_python_indexer(self, instance, conn):
|
||||
Log.note("Install Python at {{instance_id}} ({{address}})", instance_id=instance.id, address=instance.ip_address)
|
||||
self._install_python(instance, conn)
|
||||
if not conn.exists("/home/ec2-user/ActiveData-ETL/"):
|
||||
with conn.cd("/home/ec2-user"):
|
||||
conn.sudo("yum -y install git")
|
||||
conn.run("git clone https://github.com/klahnakoski/ActiveData-ETL.git")
|
||||
|
||||
if not conn.exists("/home/ec2-user/ActiveData-ETL/"):
|
||||
with conn.cd("/home/ec2-user"):
|
||||
conn.sudo("yum -y install git")
|
||||
conn.run("git clone https://github.com/klahnakoski/ActiveData-ETL.git")
|
||||
with conn.cd("/home/ec2-user/ActiveData-ETL/"):
|
||||
conn.run("git checkout push-to-es6")
|
||||
conn.sudo("yum -y install gcc") # REQUIRED FOR psutil
|
||||
conn.run("~/pypy/bin/pypy -m pip install -r requirements.txt")
|
||||
|
||||
with conn.cd("/home/ec2-user/ActiveData-ETL/"):
|
||||
conn.run("git checkout push-to-es6")
|
||||
conn.sudo("yum -y install gcc") # REQUIRED FOR psutil
|
||||
conn.run("~/pypy/bin/pypy -m pip install -r requirements.txt")
|
||||
conn.put("~/private_active_data_etl.json", "/home/ec2-user/private.json")
|
||||
|
||||
conn.put("~/private_active_data_etl.json", "/home/ec2-user/private.json")
|
||||
def _install_python(instance, conn):
|
||||
Log.note("Install Python at {{instance_id}} ({{address}})", instance_id=instance.id, address=instance.ip_address)
|
||||
if conn.exists("/usr/bin/pip"):
|
||||
pip_version = text(conn.sudo("pip --version", warn=True))
|
||||
else:
|
||||
pip_version = ""
|
||||
|
||||
def _install_python(self, instance, conn):
|
||||
Log.note("Install Python at {{instance_id}} ({{address}})", instance_id=instance.id, address=instance.ip_address)
|
||||
if conn.exists("/usr/bin/pip"):
|
||||
pip_version = conn.sudo("pip --version", warn=True)
|
||||
else:
|
||||
pip_version = ""
|
||||
if not pip_version.startswith("pip 18."):
|
||||
conn.sudo("yum -y install python2")
|
||||
conn.sudo("easy_install pip")
|
||||
# conn.sudo("rm -f /usr/bin/pip", warn=True)
|
||||
# conn.sudo("ln -s /usr/local/bin/pip /usr/bin/pip")
|
||||
# conn.sudo("pip install --upgrade pip")
|
||||
|
||||
if not pip_version.startswith("pip 9."):
|
||||
conn.sudo("yum -y install python27")
|
||||
conn.sudo("easy_install pip")
|
||||
conn.sudo("rm -f /usr/bin/pip", warn=True)
|
||||
conn.sudo("ln -s /usr/local/bin/pip /usr/bin/pip")
|
||||
conn.sudo("pip install --upgrade pip")
|
||||
def _install_pypy(instance, conn):
|
||||
Log.note("Install pypy at {{instance_id}} ({{address}})", instance_id=instance.id, address=instance.ip_address)
|
||||
|
||||
def _install_pypy(self, instance, conn):
|
||||
Log.note("Install pypy at {{instance_id}} ({{address}})", instance_id=instance.id, address=instance.ip_address)
|
||||
if not (RESOURCES / PYPY_BZ2).exists:
|
||||
Log.error("Expecting {{file}} on manager to spread to ES instances", file=(RESOURCES / PYPY_BZ2))
|
||||
|
||||
if not File(LOCAL_PYPY).exists:
|
||||
Log.error("Expecting {{file}} on manager to spread to ES instances", file=LOCAL_PYPY)
|
||||
if conn.exists("~/pypy/bin/pip"):
|
||||
return
|
||||
|
||||
if conn.exists("~/pypy/bin/pip"):
|
||||
return
|
||||
with conn.cd("/home/ec2-user/"):
|
||||
conn.put((RESOURCES / PYPY_BZ2), PYPY_BZ2)
|
||||
conn.run('tar jxf ' + PYPY_BZ2)
|
||||
conn.run("mv " + PYPY_DIR + " pypy")
|
||||
|
||||
with conn.cd("/home/ec2-user/"):
|
||||
conn.put(LOCAL_PYPY, PYPY_BZ2)
|
||||
conn.run('tar jxf ' + PYPY_BZ2)
|
||||
conn.run("mv " + PYPY_DIR + " pypy")
|
||||
conn.run("rm -fr /home/ec2-user/temp", warn=True)
|
||||
conn.run("mkdir /home/ec2-user/temp")
|
||||
with conn.cd("/home/ec2-user/temp"):
|
||||
conn.run("wget https://bootstrap.pypa.io/get-pip.py")
|
||||
conn.run("~/pypy/bin/pypy get-pip.py")
|
||||
|
||||
conn.run("rm -fr /home/ec2-user/temp", warn=True)
|
||||
conn.run("mkdir /home/ec2-user/temp")
|
||||
with conn.cd("/home/ec2-user/temp"):
|
||||
conn.run("wget https://bootstrap.pypa.io/get-pip.py")
|
||||
conn.run("~/pypy/bin/pypy get-pip.py")
|
||||
def _install_pypy_indexer(instance, conn):
|
||||
Log.note("Install indexer at {{instance_id}} ({{address}})", instance_id=instance.id, address=instance.ip_address)
|
||||
_install_pypy(instance, conn)
|
||||
|
||||
def _install_pypy_indexer(self, instance, conn):
|
||||
Log.note("Install indexer at {{instance_id}} ({{address}})", instance_id=instance.id, address=instance.ip_address)
|
||||
self._install_pypy(instance, conn)
|
||||
if not conn.exists("/home/ec2-user/ActiveData-ETL/"):
|
||||
with conn.cd("/home/ec2-user"):
|
||||
conn.sudo("yum -y install git")
|
||||
conn.run("git clone https://github.com/klahnakoski/ActiveData-ETL.git")
|
||||
|
||||
if not conn.exists("/home/ec2-user/ActiveData-ETL/"):
|
||||
with conn.cd("/home/ec2-user"):
|
||||
conn.sudo("yum -y install git")
|
||||
conn.run("git clone https://github.com/klahnakoski/ActiveData-ETL.git")
|
||||
with conn.cd("/home/ec2-user/ActiveData-ETL/"):
|
||||
conn.run("git checkout push-to-es6")
|
||||
conn.run("git pull origin push-to-es6")
|
||||
conn.sudo("yum -y install gcc") # REQUIRED FOR psutil
|
||||
conn.run("~/pypy/bin/pip install -r requirements.txt")
|
||||
|
||||
with conn.cd("/home/ec2-user/ActiveData-ETL/"):
|
||||
conn.run("git checkout push-to-es6")
|
||||
conn.sudo("yum -y install gcc") # REQUIRED FOR psutil
|
||||
conn.run("~/pypy/bin/pip install -r requirements.txt")
|
||||
conn.put("~/private_active_data_etl.json", "/home/ec2-user/private.json")
|
||||
|
||||
conn.put("~/private_active_data_etl.json", "/home/ec2-user/private.json")
|
||||
def _install_supervisor(instance, conn):
|
||||
_install_python(instance, conn)
|
||||
conn.sudo("pip install supervisor")
|
||||
|
||||
def _install_supervisor(self, instance, conn):
|
||||
Log.note("Install Supervisor-plus-Cron at {{instance_id}} ({{address}})", instance_id=instance.id, address=instance.ip_address)
|
||||
# REQUIRED FOR Python SSH
|
||||
self._install_lib("libffi-devel", conn)
|
||||
self._install_lib("openssl-devel", conn)
|
||||
self._install_lib('"Development tools"', install="groupinstall", conn=conn)
|
||||
def _install_lib(lib_name, install="install", conn=None):
|
||||
"""
|
||||
:param lib_name:
|
||||
:param install: use 'groupinstall' if you wish
|
||||
:return:
|
||||
"""
|
||||
result = conn.sudo("yum "+install+" -y "+lib_name, warn=True)
|
||||
if result.return_code != 0 and result.find("already installed and latest version") == -1:
|
||||
Log.error("problem with install of {{lib}}", lib=lib_name)
|
||||
|
||||
self._install_python(instance, conn)
|
||||
conn.sudo("pip install pyopenssl")
|
||||
conn.sudo("pip install ndg-httpsclient")
|
||||
conn.sudo("pip install pyasn1")
|
||||
conn.sudo("pip install fabric==1.10.2")
|
||||
conn.sudo("pip install requests")
|
||||
def _start_supervisor(conn):
|
||||
conn.put("./examples/config/es6_supervisor.conf", "/etc/supervisord.conf", use_sudo=True)
|
||||
|
||||
conn.sudo("pip install supervisor-plus-cron")
|
||||
# START DAEMON (OR THROW ERROR IF RUNNING ALREADY)
|
||||
conn.sudo("supervisord -c /etc/supervisord.conf", warn=True)
|
||||
conn.sudo("supervisorctl reread")
|
||||
conn.sudo("supervisorctl update")
|
||||
|
||||
def _install_lib(self, lib_name, install="install", conn=None):
|
||||
"""
|
||||
:param lib_name:
|
||||
:param install: use 'groupinstall' if you wish
|
||||
:return:
|
||||
"""
|
||||
result = conn.sudo("yum "+install+" -y "+lib_name, warn=True)
|
||||
if result.return_code != 0 and result.find("already installed and latest version") == -1:
|
||||
Log.error("problem with install of {{lib}}", lib=lib_name)
|
||||
|
||||
def _start_supervisor(self, conn):
|
||||
conn.put("./examples/config/es6_supervisor.conf", "/etc/supervisord.conf", use_sudo=True)
|
||||
|
||||
# START DAEMON (OR THROW ERROR IF RUNNING ALREADY)
|
||||
conn.sudo("supervisord -c /etc/supervisord.conf", warn=True)
|
||||
conn.sudo("supervisorctl reread")
|
||||
conn.sudo("supervisorctl update")
|
||||
|
|
|
|||
|
|
@ -9,12 +9,12 @@
|
|||
from __future__ import division
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import mo_math
|
||||
from mo_fabric import Connection
|
||||
from mo_files import File
|
||||
from mo_kwargs import override
|
||||
from mo_logs import Log, constants, startup
|
||||
from mo_logs.strings import between
|
||||
from mo_math import Math
|
||||
from mo_times import Date
|
||||
from pyLibrary import aws
|
||||
from spot.instance_manager import InstanceManager
|
||||
|
|
@ -43,8 +43,7 @@ class ETL(InstanceManager):
|
|||
|
||||
if current_utility < pending / 20:
|
||||
# INCREASE
|
||||
# ENSURE THERE IS PLENTY OF WORK BEFORE MACHINE IS DEPLOYED
|
||||
return max(minimum, Math.ceiling(pending / 20))
|
||||
return max(minimum, mo_math.ceiling(pending / 20)) # ENSURE THERE IS PLENTY OF WORK BEFORE MACHINE IS DEPLOYED
|
||||
else:
|
||||
# DECREASE
|
||||
target = max(minimum, min(current_utility, pending*2))
|
||||
|
|
|
|||
|
|
@ -1,2 +0,0 @@
|
|||
SET PYTHONPATH=.;vendor
|
||||
python spot\spot_manager.py --settings=./examples/config/es_settings.json
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
export PYTHONPATH=.:vendor
|
||||
cd ~/SpotManager
|
||||
python spot/spot_manager.py --settings=./examples/config/es_settings.json
|
||||
|
|
@ -6,40 +6,46 @@
|
|||
#
|
||||
# Author: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
from __future__ import division
|
||||
from __future__ import unicode_literals
|
||||
from __future__ import division, unicode_literals
|
||||
|
||||
from copy import copy
|
||||
|
||||
import boto
|
||||
import boto.ec2
|
||||
import boto.vpc
|
||||
from boto.ec2.blockdevicemapping import BlockDeviceType, BlockDeviceMapping
|
||||
from boto.ec2.networkinterface import NetworkInterfaceSpecification, NetworkInterfaceCollection
|
||||
from boto.ec2.blockdevicemapping import BlockDeviceMapping, BlockDeviceType
|
||||
from boto.ec2.networkinterface import NetworkInterfaceCollection, NetworkInterfaceSpecification
|
||||
from boto.utils import ISO8601
|
||||
|
||||
import mo_math
|
||||
from jx_python import jx
|
||||
from jx_python.containers.list_usingPythonList import ListContainer
|
||||
from mo_collections import UniqueIndex
|
||||
from mo_dots import unwrap, coalesce, wrap, listwrap, FlatList, Data
|
||||
from mo_dots import Data, FlatList, coalesce, listwrap, unwrap, wrap, Null
|
||||
from mo_dots.objects import datawrap
|
||||
from mo_files import File
|
||||
from mo_future import text_type
|
||||
from mo_future import text
|
||||
from mo_http import http
|
||||
from mo_json import value2json
|
||||
from mo_kwargs import override
|
||||
from mo_logs import Log, startup, Except, constants
|
||||
from mo_logs import Except, Log, constants, startup
|
||||
from mo_logs.startup import SingleInstance
|
||||
from mo_math import Math, MAX, SUM
|
||||
from mo_threads import Lock, Thread, Till, Signal
|
||||
from mo_math import MAX, SUM
|
||||
from mo_threads import Lock, Signal, Thread, Till
|
||||
from mo_threads.threads import MAIN_THREAD
|
||||
from mo_times import MINUTE, Date, Duration, Timer, DAY, WEEK, SECOND, HOUR
|
||||
from mo_times import DAY, Date, Duration, HOUR, MINUTE, SECOND, Timer, WEEK
|
||||
from pyLibrary import convert
|
||||
from pyLibrary.meta import cache, new_instance
|
||||
|
||||
_please_import = http
|
||||
|
||||
ENABLE_SIDE_EFFECTS = True
|
||||
ALLOW_SHUTDOWN = False
|
||||
DEBUG_PRICING = True
|
||||
TIME_FROM_RUNNING_TO_LOGIN = 7 * MINUTE
|
||||
ERROR_ON_CALL_TO_SETUP = "Problem with setup()"
|
||||
DELAY_BEFORE_SETUP = 1 * MINUTE # PROBLEM WITH CONNECTING ONLY HAPPENS WITH BIGGER ES MACHINES
|
||||
CAPACITY_NOT_AVAILABLE_RETRY = Duration("day") # SOME MACHINES ARE NOT AVAILABLE
|
||||
|
||||
|
||||
class SpotManager(object):
|
||||
|
|
@ -57,7 +63,9 @@ class SpotManager(object):
|
|||
self.price_locker = Lock()
|
||||
self.prices = None
|
||||
self.price_lookup = None
|
||||
self.done_spot_requests = Signal()
|
||||
self.no_capacity = {}
|
||||
self.no_capacity_file = File(kwargs.price_file).parent / "no capacity.json"
|
||||
self.done_making_new_spot_requests = Signal()
|
||||
self.net_new_locker = Lock()
|
||||
self.net_new_spot_requests = UniqueIndex(("id",)) # SPOT REQUESTS FOR THIS SESSION
|
||||
self.watcher = None
|
||||
|
|
@ -119,12 +127,12 @@ class SpotManager(object):
|
|||
|
||||
if net_new_utility < 0:
|
||||
if self.settings.allowed_overage:
|
||||
net_new_utility = Math.min(net_new_utility + self.settings.allowed_overage * utility_required, 0)
|
||||
net_new_utility = mo_math.min(net_new_utility + self.settings.allowed_overage * utility_required, 0)
|
||||
|
||||
net_new_utility = self.remove_instances(net_new_utility)
|
||||
|
||||
if net_new_utility > 0:
|
||||
net_new_utility = Math.min(net_new_utility, self.settings.max_new_utility)
|
||||
net_new_utility = mo_math.min(net_new_utility, self.settings.max_new_utility)
|
||||
net_new_utility, remaining_budget = self.add_instances(net_new_utility, remaining_budget)
|
||||
|
||||
if net_new_utility > 0:
|
||||
|
|
@ -142,7 +150,7 @@ class SpotManager(object):
|
|||
req.add_tag("Name", self.settings.ec2.instance.name)
|
||||
|
||||
Log.note("All requests for new utility have been made")
|
||||
self.done_spot_requests.go()
|
||||
self.done_making_new_spot_requests.go()
|
||||
|
||||
def add_instances(self, net_new_utility, remaining_budget):
|
||||
prices = self.pricing()
|
||||
|
|
@ -164,7 +172,7 @@ class SpotManager(object):
|
|||
|
||||
# DO NOT BID HIGHER THAN WHAT WE ARE WILLING TO PAY
|
||||
max_acceptable_price = p.type.utility * self.settings.max_utility_price + p.type.discount
|
||||
max_bid = Math.min(p.higher_price, max_acceptable_price, remaining_budget)
|
||||
max_bid = mo_math.min(p.higher_price, max_acceptable_price, remaining_budget)
|
||||
min_bid = p.price_80
|
||||
|
||||
if min_bid > max_acceptable_price:
|
||||
|
|
@ -186,15 +194,15 @@ class SpotManager(object):
|
|||
elif min_bid > max_bid:
|
||||
Log.error("not expected")
|
||||
|
||||
naive_number_needed = int(Math.round(float(net_new_utility) / float(p.type.utility), decimal=0))
|
||||
naive_number_needed = int(mo_math.round(float(net_new_utility) / float(p.type.utility), decimal=0))
|
||||
limit_total = None
|
||||
if self.settings.max_percent_per_type < 1:
|
||||
current_count = sum(1 for a in self.active if a.launch_specification.instance_type == p.type.instance_type and a.launch_specification.placement == p.availability_zone)
|
||||
all_count = sum(1 for a in self.active if a.launch_specification.placement == p.availability_zone)
|
||||
all_count = max(all_count, naive_number_needed)
|
||||
limit_total = int(Math.floor((all_count * self.settings.max_percent_per_type - current_count) / (1 - self.settings.max_percent_per_type)))
|
||||
limit_total = int(mo_math.floor((all_count * self.settings.max_percent_per_type - current_count) / (1 - self.settings.max_percent_per_type)))
|
||||
|
||||
num = Math.min(naive_number_needed, limit_total, self.settings.max_requests_per_type)
|
||||
num = mo_math.min(naive_number_needed, limit_total, self.settings.max_requests_per_type)
|
||||
if num < 0:
|
||||
Log.note(
|
||||
"{{type}} is over {{limit|percent}} of instances, no more requested",
|
||||
|
|
@ -203,10 +211,10 @@ class SpotManager(object):
|
|||
)
|
||||
continue
|
||||
elif num == 1:
|
||||
min_bid = Math.min(Math.max(p.current_price * 1.1, min_bid), max_acceptable_price)
|
||||
min_bid = mo_math.min(mo_math.max(p.current_price * 1.1, min_bid), max_acceptable_price)
|
||||
price_interval = 0
|
||||
else:
|
||||
price_interval = Math.min(min_bid / 10, (max_bid - min_bid) / (num - 1))
|
||||
price_interval = mo_math.min(min_bid / 10, (max_bid - min_bid) / (num - 1))
|
||||
|
||||
for i in range(num):
|
||||
bid_per_machine = min_bid + (i * price_interval)
|
||||
|
|
@ -227,6 +235,15 @@ class SpotManager(object):
|
|||
)
|
||||
continue
|
||||
|
||||
last_no_capacity_message = self.no_capacity.get(p.type.instance_type, Null)
|
||||
if last_no_capacity_message > Date.now() - CAPACITY_NOT_AVAILABLE_RETRY:
|
||||
Log.note(
|
||||
"Did not bid on {{type}}: \"No capacity\" last seen at {{last_time|datetime}}",
|
||||
type=p.type.instance_type,
|
||||
last_time=last_no_capacity_message
|
||||
)
|
||||
continue
|
||||
|
||||
try:
|
||||
if self.settings.ec2.request.count == None or self.settings.ec2.request.count != 1:
|
||||
Log.error("Spot Manager can only request machine one-at-a-time")
|
||||
|
|
@ -288,7 +305,7 @@ class SpotManager(object):
|
|||
Log.note("Shutdown {{instances}}", instances=remove_list.id)
|
||||
remove_threads = [
|
||||
Thread.run(
|
||||
"teardown for " + text_type(i.id),
|
||||
"teardown for " + text(i.id),
|
||||
self.instance_manager.teardown,
|
||||
i
|
||||
)
|
||||
|
|
@ -352,23 +369,24 @@ class SpotManager(object):
|
|||
|
||||
# SEND SHUTDOWN TO EACH INSTANCE
|
||||
Log.warning("Shutdown {{instances}} to save money!", instances=remove_list.id)
|
||||
for g, removals in jx.groupby(remove_list, size=20):
|
||||
for i, t in [
|
||||
(i, Thread.run("teardown " + i.id, self.instance_manager.teardown, i, please_stop=False))
|
||||
for i in removals
|
||||
]:
|
||||
try:
|
||||
t.join()
|
||||
except Exception:
|
||||
Log.note("Problem with shutdown of {{id}}", id=i.id)
|
||||
if ALLOW_SHUTDOWN:
|
||||
for g, removals in jx.groupby(remove_list, size=20):
|
||||
for i, t in [
|
||||
(i, Thread.run("teardown " + i.id, self.instance_manager.teardown, i, please_stop=False))
|
||||
for i in removals
|
||||
]:
|
||||
try:
|
||||
t.join()
|
||||
except Exception:
|
||||
Log.note("Problem with shutdown of {{id}}", id=i.id)
|
||||
|
||||
remove_spot_requests.extend(remove_list.spot_instance_request_id)
|
||||
remove_spot_requests.extend(remove_list.spot_instance_request_id)
|
||||
|
||||
# TERMINATE INSTANCES
|
||||
self.ec2_conn.terminate_instances(instance_ids=remove_list.id)
|
||||
# TERMINATE INSTANCES
|
||||
self.ec2_conn.terminate_instances(instance_ids=remove_list.id)
|
||||
|
||||
# TERMINATE SPOT REQUESTS
|
||||
self.ec2_conn.cancel_spot_instance_requests(request_ids=remove_spot_requests)
|
||||
# TERMINATE SPOT REQUESTS
|
||||
self.ec2_conn.cancel_spot_instance_requests(request_ids=remove_spot_requests)
|
||||
return remaining_budget, net_new_utility
|
||||
|
||||
@cache(duration=5 * SECOND)
|
||||
|
|
@ -389,19 +407,54 @@ class SpotManager(object):
|
|||
return wrap(output)
|
||||
|
||||
def _start_life_cycle_watcher(self):
|
||||
failed_locker = Lock()
|
||||
failed_attempts = Data()
|
||||
|
||||
def track_setup(
|
||||
instance_setup_function,
|
||||
request,
|
||||
instance, # THE boto INSTANCE OBJECT FOR THE MACHINE TO SETUP
|
||||
utility, # THE utility OBJECT FOUND IN CONFIG
|
||||
please_stop
|
||||
):
|
||||
try:
|
||||
instance_setup_function(instance, utility, please_stop)
|
||||
instance.add_tag("Name", self.settings.ec2.instance.name + " (running)")
|
||||
with self.net_new_locker:
|
||||
self.net_new_spot_requests.remove(request.id)
|
||||
except Exception as e:
|
||||
e = Except.wrap(e)
|
||||
instance.add_tag("Name", "")
|
||||
with failed_locker:
|
||||
failed_attempts[request.id] += [e]
|
||||
if "Can not setup unknown " in e:
|
||||
Log.warning("Unexpected failure on startup", instance_id=instance.id, cause=e)
|
||||
elif ERROR_ON_CALL_TO_SETUP in e:
|
||||
with failed_locker:
|
||||
causes = failed_attempts[request.id]
|
||||
if len(causes) > 2:
|
||||
Log.warning("Problem with setup() of {{instance_id}}", instance_id=instance.id, cause=causes)
|
||||
else:
|
||||
Log.warning("Unexpected failure on startup", instance_id=instance.id, cause=e)
|
||||
|
||||
def life_cycle_watcher(please_stop):
|
||||
failed_attempts = Data()
|
||||
bad_requests = Data()
|
||||
setup_threads = []
|
||||
last_get = Date.now()
|
||||
|
||||
while not please_stop:
|
||||
spot_requests = self._get_managed_spot_requests()
|
||||
last_get = Date.now()
|
||||
instances = wrap({i.id: i for r in self.ec2_conn.get_all_instances() for i in r.instances})
|
||||
# INSTANCES THAT REQUIRE SETUP
|
||||
time_to_stop_trying = {}
|
||||
please_setup = [
|
||||
(i, r) for i, r in [(instances[r.instance_id], r) for r in spot_requests]
|
||||
if i.id and not i.tags.get("Name") and i._state.name == "running" and Date.now() > Date(i.launch_time) + DELAY_BEFORE_SETUP
|
||||
if i.id
|
||||
and (
|
||||
not i.tags.get("Name") or i.tags.get("Name") == self.settings.ec2.instance.name + " (setup)"
|
||||
)
|
||||
and i._state.name == "running"
|
||||
and Date.now() > Date(i.launch_time) + DELAY_BEFORE_SETUP
|
||||
]
|
||||
|
||||
for i, r in please_setup:
|
||||
|
|
@ -412,7 +465,7 @@ class SpotManager(object):
|
|||
self.ec2_conn.terminate_instances(instance_ids=[i.id])
|
||||
with self.net_new_locker:
|
||||
self.net_new_spot_requests.remove(r.id)
|
||||
Log.warning("Problem with setup of {{instance_id}}. Time is up. Instance TERMINATED!", instance_id=i.id, cause=e)
|
||||
Log.warning("Problem with setup of {{instance_id}}. Time is up. Instance TERMINATED!", instance_id=i.id)
|
||||
continue
|
||||
|
||||
try:
|
||||
|
|
@ -427,57 +480,35 @@ class SpotManager(object):
|
|||
|
||||
i.markup = p
|
||||
i.add_tag("Name", self.settings.ec2.instance.name + " (setup)")
|
||||
setup_threads.append((i, r, Thread.run(
|
||||
"setup for " + text_type(i.id),
|
||||
setup_threads.append(Thread.run(
|
||||
"setup for " + text(i.id),
|
||||
track_setup,
|
||||
self.instance_manager.setup,
|
||||
r,
|
||||
i,
|
||||
p
|
||||
)))
|
||||
))
|
||||
except Exception as e:
|
||||
i.add_tag("Name", "")
|
||||
Log.warning("Unexpected failure on startup", instance_id=i.id, cause=e)
|
||||
|
||||
please_join = [(i, r, t) for i, r, t in setup_threads if t.stopped]
|
||||
if please_join:
|
||||
Log.note("{{num}} threads have stopped", num=len(please_join))
|
||||
for i, r, t in please_join:
|
||||
try:
|
||||
t.join()
|
||||
setup_threads.remove((i, r, t))
|
||||
i.add_tag("Name", self.settings.ec2.instance.name + " (running)")
|
||||
with self.net_new_locker:
|
||||
self.net_new_spot_requests.remove(r.id)
|
||||
except Exception as e:
|
||||
e = Except.wrap(e)
|
||||
setup_threads.remove((i, r, t))
|
||||
i.add_tag("Name", "")
|
||||
failed_attempts[r.id] += [e]
|
||||
if "Can not setup unknown " in e:
|
||||
Log.warning("Unexpected failure on startup", instance_id=i.id, cause=e)
|
||||
elif ERROR_ON_CALL_TO_SETUP in e:
|
||||
if len(failed_attempts[r.id]) > 2:
|
||||
Log.warning("Problem with setup() of {{instance_id}}", instance_id=i.id, cause=failed_attempts[r.id])
|
||||
else:
|
||||
Log.warning("Unexpected failure on startup", instance_id=i.id, cause=e)
|
||||
|
||||
if Date.now() - last_get > 5 * SECOND:
|
||||
# REFRESH STALE
|
||||
spot_requests = self._get_managed_spot_requests()
|
||||
last_get = Date.now()
|
||||
|
||||
pending = wrap([r for r in spot_requests if r.status.code in PENDING_STATUS_CODES])
|
||||
give_up = wrap([r for r in spot_requests if r.status.code in PROBABLY_NOT_FOR_A_WHILE | TERMINATED_STATUS_CODES])
|
||||
give_up = wrap([r for r in spot_requests if (r.status.code in PROBABLY_NOT_FOR_A_WHILE | TERMINATED_STATUS_CODES) and r.id not in bad_requests])
|
||||
ignore = wrap([r for r in spot_requests if r.status.code in MIGHT_HAPPEN]) # MIGHT HAPPEN, BUT NO NEED TO WAIT FOR IT
|
||||
|
||||
if self.done_spot_requests:
|
||||
if self.done_making_new_spot_requests:
|
||||
with self.net_new_locker:
|
||||
expired = Date.now() - self.settings.run_interval + 2 * MINUTE
|
||||
for ii in list(self.net_new_spot_requests):
|
||||
if Date(ii.create_time) < expired:
|
||||
## SOMETIMES REQUESTS NEVER GET INTO THE MAIN LIST OF REQUESTS
|
||||
# SOMETIMES REQUESTS NEVER GET INTO THE MAIN LIST OF REQUESTS
|
||||
self.net_new_spot_requests.remove(ii)
|
||||
for g in give_up:
|
||||
self.net_new_spot_requests.remove(g.id)
|
||||
|
||||
for g in ignore:
|
||||
self.net_new_spot_requests.remove(g.id)
|
||||
pending = UniqueIndex(("id",), data=pending)
|
||||
|
|
@ -487,17 +518,35 @@ class SpotManager(object):
|
|||
self.ec2_conn.cancel_spot_instance_requests(request_ids=give_up.id)
|
||||
Log.note("Cancelled spot requests {{spots}}, {{reasons}}", spots=give_up.id, reasons=give_up.status.code)
|
||||
|
||||
if not pending and not time_to_stop_trying and self.done_spot_requests and not setup_threads:
|
||||
for g in give_up:
|
||||
bad_requests[g.id] += 1
|
||||
if g.id in self.net_new_spot_requests:
|
||||
self.net_new_spot_requests.remove(g.id)
|
||||
if g.status.code == "capacity-not-available":
|
||||
self.no_capacity[g.launch_specification.instance_type] = Date.now()
|
||||
if g.status.code == "bad-parameters":
|
||||
self.no_capacity[g.launch_specification.instance_type] = Date.now()
|
||||
Log.warning("bad parameters while requesting type {{type}}", type=g.launch_specification.instance_type)
|
||||
|
||||
if not pending and self.done_making_new_spot_requests:
|
||||
Log.note("No more pending spot requests")
|
||||
please_stop.go()
|
||||
break
|
||||
elif setup_threads:
|
||||
Log.note("waiting for setup of {{num}} instances", num=len(setup_threads))
|
||||
elif pending:
|
||||
Log.note("waiting for spot requests: {{pending}}", pending=[p.id for p in pending])
|
||||
|
||||
(Till(seconds=10) | please_stop).wait()
|
||||
|
||||
with Timer("Save no capacity to file"):
|
||||
table = [
|
||||
{"instance_type": k, "last_failure": v}
|
||||
for k, v in self.no_capacity.items()
|
||||
]
|
||||
self.no_capacity_file.write(value2json(table, pretty=True))
|
||||
|
||||
# WAIT FOR SETUP TO COMPLETE
|
||||
for t in setup_threads:
|
||||
t.join()
|
||||
|
||||
Log.note("life cycle watcher has stopped")
|
||||
|
||||
# Log.warning("lifecycle watcher is disabled")
|
||||
|
|
@ -518,6 +567,7 @@ class SpotManager(object):
|
|||
|
||||
@override
|
||||
def _request_spot_instances(self, price, availability_zone_group, instance_type, kwargs):
|
||||
kwargs.self = None
|
||||
kwargs.kwargs = None
|
||||
|
||||
# m3 INSTANCES ARE NOT ALLOWED PLACEMENT GROUP
|
||||
|
|
@ -549,7 +599,7 @@ class SpotManager(object):
|
|||
for i in range(num_ephemeral_volumes):
|
||||
letter = convert.ascii2char(98 + i) # START AT "b"
|
||||
kwargs.block_device_map["/dev/sd" + letter] = BlockDeviceType(
|
||||
ephemeral_name='ephemeral' + text_type(i),
|
||||
ephemeral_name='ephemeral' + text(i),
|
||||
delete_on_termination=True
|
||||
)
|
||||
|
||||
|
|
@ -658,6 +708,17 @@ class SpotManager(object):
|
|||
return self.prices
|
||||
|
||||
def _get_spot_prices_from_aws(self):
|
||||
with Timer("Read no capacity file"):
|
||||
try:
|
||||
# FILE IS LIST OF {instance_type, last_failure} OBJECTS
|
||||
content = self.no_capacity_file.read()
|
||||
self.no_capacity = dict(
|
||||
(r.instance_type, r.last_failure)
|
||||
for r in convert.json2value(content, flexible=False, leaves=False)
|
||||
)
|
||||
except Exception as e:
|
||||
self.no_capacity = {}
|
||||
|
||||
with Timer("Read pricing file"):
|
||||
try:
|
||||
content = File(self.settings.price_file).read()
|
||||
|
|
@ -739,7 +800,7 @@ def find_higher(candidates, reference):
|
|||
|
||||
|
||||
TERMINATED_STATUS_CODES = {
|
||||
"marked-for-termination", # AS GOOD AS DEAD
|
||||
"marked-for-termination", # AS GOOD AS DEAD
|
||||
"capacity-oversubscribed",
|
||||
"capacity-not-available",
|
||||
"instance-terminated-capacity-oversubscribed",
|
||||
|
|
@ -762,8 +823,6 @@ PROBABLY_NOT_FOR_A_WHILE = {
|
|||
"placement-group-constraint",
|
||||
"price-too-low"
|
||||
}
|
||||
|
||||
|
||||
RUNNING_STATUS_CODES = {
|
||||
"fulfilled",
|
||||
"request-canceled-and-instance-running"
|
||||
|
|
@ -773,9 +832,9 @@ RUNNING_STATUS_CODES = {
|
|||
def main():
|
||||
try:
|
||||
settings = startup.read_settings()
|
||||
constants.set(settings.constants)
|
||||
Log.start(settings.debug)
|
||||
with SingleInstance(flavor_id=settings.args.filename):
|
||||
constants.set(settings.constants)
|
||||
settings.run_interval = Duration(settings.run_interval)
|
||||
for u in settings.utility:
|
||||
u.discount = coalesce(u.discount, 0)
|
||||
|
|
@ -804,31 +863,35 @@ def main():
|
|||
ephemeral_storage = {
|
||||
"c1.medium": {"num": 1, "size": 350},
|
||||
"c1.xlarge": {"num": 4, "size": 1680},
|
||||
|
||||
"c3.2xlarge": {"num": 2, "size": 160},
|
||||
"c3.4xlarge": {"num": 2, "size": 320},
|
||||
"c3.8xlarge": {"num": 2, "size": 640},
|
||||
"c3.large": {"num": 2, "size": 32},
|
||||
"c3.xlarge": {"num": 2, "size": 80},
|
||||
|
||||
"c4.2xlarge": {"num": 0, "size": 0},
|
||||
"c4.4xlarge": {"num": 0, "size": 0},
|
||||
"c4.8xlarge": {"num": 0, "size": 0},
|
||||
"c4.large": {"num": 0, "size": 0},
|
||||
"c4.xlarge": {"num": 0, "size": 0},
|
||||
|
||||
"c5.large":{"num": 0, "size": 0},
|
||||
"c5.xlarge":{"num": 0, "size": 0},
|
||||
"c5.2xlarge":{"num": 0, "size": 0},
|
||||
"c5.4xlarge":{"num": 0, "size": 0},
|
||||
"c5.9xlarge":{"num": 0, "size": 0},
|
||||
"c5.18xlarge":{"num": 0, "size": 0},
|
||||
|
||||
"cc2.8xlarge": {"num": 4, "size": 3360},
|
||||
"cg1.4xlarge": {"num": 2, "size": 1680},
|
||||
"cr1.8xlarge": {"num": 2, "size": 240},
|
||||
|
||||
"d2.2xlarge": {"num": 6, "size": 12000},
|
||||
"d2.4xlarge": {"num": 12, "size": 24000},
|
||||
"d2.8xlarge": {"num": 24, "size": 48000},
|
||||
"d2.xlarge": {"num": 3, "size": 6000},
|
||||
"f1.16xlarge": {"num": 0, "size": 0},
|
||||
"f1.2xlarge": {"num": 0, "size": 0},
|
||||
|
||||
"g2.2xlarge": {"num": 1, "size": 60},
|
||||
"g2.8xlarge": {"num": 2, "size": 240},
|
||||
"h1.2xlarge": {"num": 1, "size": 2000},
|
||||
|
|
@ -842,19 +905,18 @@ ephemeral_storage = {
|
|||
"i2.4xlarge": {"num": 4, "size": 3200},
|
||||
"i2.8xlarge": {"num": 8, "size": 6400},
|
||||
"i2.xlarge": {"num": 1, "size": 800},
|
||||
"i3.16xlarge": {"num": 8, "size": 15200},
|
||||
|
||||
"i3.large": {"num": 1, "size": 475},
|
||||
"i3.xlarge": {"num": 1, "size": 950},
|
||||
"i3.2xlarge": {"num": 1, "size": 1900},
|
||||
"i3.4xlarge": {"num": 2, "size": 3800},
|
||||
"i3.8xlarge": {"num": 4, "size": 7600},
|
||||
"i3.large": {"num": 1, "size": 475},
|
||||
"i3.xlarge": {"num": 1, "size": 950},
|
||||
"m1.large": {"num": 2, "size": 840},
|
||||
"m1.medium": {"num": 1, "size": 410},
|
||||
"m1.small": {"num": 1, "size": 160},
|
||||
"m1.xlarge": {"num": 4, "size": 1680},
|
||||
"m2.2xlarge": {"num": 1, "size": 850},
|
||||
"m2.4xlarge": {"num": 2, "size": 1680},
|
||||
"m2.xlarge": {"num": 1, "size": 420},
|
||||
"i3.16xlarge": {"num": 8, "size": 15200},
|
||||
|
||||
"f1.2xlarge": {"num": 1, "size": 470},
|
||||
"f1.4xlarge": {"num": 1, "size": 940},
|
||||
"f1.16xlarge": {"num": 4, "size": 940},
|
||||
|
||||
"m3.2xlarge": {"num": 2, "size": 160},
|
||||
"m3.large": {"num": 1, "size": 32},
|
||||
"m3.medium": {"num": 1, "size": 4},
|
||||
|
|
@ -865,6 +927,14 @@ ephemeral_storage = {
|
|||
"m4.4xlarge": {"num": 0, "size": 0},
|
||||
"m4.large": {"num": 0, "size": 0},
|
||||
"m4.xlarge": {"num": 0, "size": 0},
|
||||
|
||||
"m5d.large": {"num": 1, "size": 75},
|
||||
"m5d.xlarge": {"num": 1, "size": 150},
|
||||
"m5d.2xlarge": {"num": 1, "size": 300},
|
||||
"m5d.4xlarge": {"num": 2, "size": 300},
|
||||
"m5d.12xlarge": {"num": 2, "size": 900},
|
||||
"m5d.24xlarge": {"num": 4, "size": 900},
|
||||
|
||||
"p2.16xlarge": {"num": 0, "size": 0},
|
||||
"p2.8xlarge": {"num": 0, "size": 0},
|
||||
"p2.xlarge": {"num": 0, "size": 0},
|
||||
|
|
@ -879,6 +949,14 @@ ephemeral_storage = {
|
|||
"r4.8xlarge": {"num": 0, "size": 0},
|
||||
"r4.large": {"num": 0, "size": 0},
|
||||
"r4.xlarge": {"num": 0, "size": 0},
|
||||
|
||||
"r5d.large": {"num": 1, "size": 75},
|
||||
"r5d.xlarge": {"num": 1, "size": 150},
|
||||
"r5d.2xlarge": {"num": 1, "size": 300},
|
||||
"r5d.4xlarge": {"num": 2, "size": 300},
|
||||
"r5d.12xlarge": {"num": 2, "size": 900},
|
||||
"r5d.24xlarge": {"num": 4, "size": 900},
|
||||
|
||||
"t1.micro": {"num": 0, "size": 0},
|
||||
"t2.2xlarge": {"num": 0, "size": 0},
|
||||
"t2.large": {"num": 0, "size": 0},
|
||||
|
|
@ -887,6 +965,28 @@ ephemeral_storage = {
|
|||
"t2.nano": {"num": 0, "size": 0},
|
||||
"t2.small": {"num": 0, "size": 0},
|
||||
"t2.xlarge": {"num": 0, "size": 0},
|
||||
|
||||
"c5d.large": {"num": 1, "size": 50},
|
||||
"c5d.xlarge": {"num": 1, "size": 100},
|
||||
"c5d.2xlarge": {"num": 1, "size": 200},
|
||||
"c5d.4xlarge": {"num": 1, "size": 400},
|
||||
"c5d.9xlarge": {"num": 1, "size": 900},
|
||||
"c5d.18xlarge": {"num": 2, "size": 900},
|
||||
|
||||
"x1e.xlarge": {"num": 1, "size": 120},
|
||||
"x1e.2xlarge": {"num": 1, "size": 240},
|
||||
"x1e.4xlarge": {"num": 1, "size": 480},
|
||||
"x1e.8xlarge": {"num": 1, "size": 960},
|
||||
"x1e.16xlarge": {"num": 1, "size": 1920},
|
||||
"x1e.32xlarge": {"num": 2, "size": 1920},
|
||||
|
||||
"z1d.large": {"num": 1, "size": 75},
|
||||
"z1d.xlarge": {"num": 1, "size": 150},
|
||||
"z1d.2xlarge": {"num": 1, "size": 300},
|
||||
"z1d.3xlarge": {"num": 1, "size": 450},
|
||||
"z1d.6xlarge": {"num": 1, "size": 900},
|
||||
"z1d.12xlarge": {"num": 1, "size": 900},
|
||||
|
||||
"x1.16xlarge": {"num": 1, "size": 1920},
|
||||
"x1.32xlarge": {"num": 2, "size": 3840}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,21 +5,26 @@
|
|||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Author: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import unicode_literals
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from uuid import uuid4
|
||||
|
||||
from mo_dots import wrap, coalesce, listwrap
|
||||
from mo_future import text_type
|
||||
from mo_json import value2json
|
||||
from jx_base.expressions import jx_expression
|
||||
from jx_python.expressions import Literal, Python
|
||||
from mo_dots import coalesce, listwrap, wrap
|
||||
from mo_dots.datas import register_data
|
||||
from mo_dots.lists import last
|
||||
from mo_future import is_text, text
|
||||
from mo_json import value2json, true, false, null
|
||||
from mo_logs import Log
|
||||
from mo_logs.strings import expand_template, quote
|
||||
|
||||
|
||||
ENABLE_CONSTRAINTS = True
|
||||
|
||||
|
||||
def generateGuid():
|
||||
"""Gets a random GUID.
|
||||
Note: python's UUID generation library is used here.
|
||||
|
|
@ -32,7 +37,7 @@ def generateGuid():
|
|||
print(a)
|
||||
print(uuid.UUID(a).hex)
|
||||
"""
|
||||
return text_type(uuid4())
|
||||
return text(uuid4())
|
||||
|
||||
|
||||
def _exec(code, name):
|
||||
|
|
@ -46,7 +51,7 @@ def _exec(code, name):
|
|||
Log.error("Can not make class\n{{code}}", code=code, cause=e)
|
||||
|
||||
|
||||
_ = listwrap
|
||||
_ = listwrap, last, true, false, null
|
||||
|
||||
|
||||
def DataClass(name, columns, constraint=None):
|
||||
|
|
@ -72,18 +77,26 @@ def DataClass(name, columns, constraint=None):
|
|||
:return: The class that has been created
|
||||
"""
|
||||
|
||||
from jx_python.expressions import jx_expression
|
||||
|
||||
columns = wrap([{"name": c, "required": True, "nulls": False, "type": object} if isinstance(c, text_type) else c for c in columns])
|
||||
columns = wrap(
|
||||
[
|
||||
{"name": c, "required": True, "nulls": False, "type": object}
|
||||
if is_text(c)
|
||||
else c
|
||||
for c in columns
|
||||
]
|
||||
)
|
||||
slots = columns.name
|
||||
required = wrap(filter(lambda c: c.required and not c.nulls and not c.default, columns)).name
|
||||
required = wrap(
|
||||
filter(lambda c: c.required and not c.nulls and not c.default, columns)
|
||||
).name
|
||||
nulls = wrap(filter(lambda c: c.nulls, columns)).name
|
||||
defaults = {c.name: coalesce(c.default, None) for c in columns}
|
||||
types = {c.name: coalesce(c.type, object) for c in columns}
|
||||
types = {c.name: coalesce(c.jx_type, object) for c in columns}
|
||||
|
||||
code = expand_template(
|
||||
"""
|
||||
"""
|
||||
from __future__ import unicode_literals
|
||||
from mo_future import is_text, is_binary
|
||||
from collections import Mapping
|
||||
|
||||
meta = None
|
||||
|
|
@ -95,10 +108,16 @@ class {{class_name}}(Mapping):
|
|||
|
||||
|
||||
def _constraint(row, rownum, rows):
|
||||
try:
|
||||
return {{constraint_expr}}
|
||||
except Exception as e:
|
||||
return False
|
||||
code = {{constraint_expr|quote}}
|
||||
if {{constraint_expr}}:
|
||||
return
|
||||
Log.error(
|
||||
"constraint\\n{" + "{code}}\\nnot satisfied {" + "{expect}}\\n{" + "{value|indent}}",
|
||||
code={{constraint_expr|quote}},
|
||||
expect={{constraint}},
|
||||
value=row,
|
||||
cause=e
|
||||
)
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
if not kwargs:
|
||||
|
|
@ -115,8 +134,7 @@ class {{class_name}}(Mapping):
|
|||
if illegal:
|
||||
Log.error("{"+"{names}} are not a valid properties", names=illegal)
|
||||
|
||||
if not self._constraint(0, [self]):
|
||||
Log.error("constraint not satisfied {"+"{expect}}\\n{"+"{value|indent}}", expect={{constraint}}, value=self)
|
||||
self._constraint(0, [self])
|
||||
|
||||
def __getitem__(self, item):
|
||||
return getattr(self, item)
|
||||
|
|
@ -128,9 +146,12 @@ class {{class_name}}(Mapping):
|
|||
def __setattr__(self, item, value):
|
||||
if item not in {{slots}}:
|
||||
Log.error("{"+"{item|quote}} not valid attribute", item=item)
|
||||
|
||||
if value==None and item in {{required}}:
|
||||
Log.error("Expecting property {"+"{item}}", item=item)
|
||||
|
||||
object.__setattr__(self, item, value)
|
||||
if not self._constraint(0, [self]):
|
||||
Log.error("constraint not satisfied {"+"{expect}}\\n{"+"{value|indent}}", expect={{constraint}}, value=self)
|
||||
self._constraint(0, [self])
|
||||
|
||||
def __getattr__(self, item):
|
||||
Log.error("{"+"{item|quote}} not valid attribute", item=item)
|
||||
|
|
@ -170,63 +191,72 @@ class {{class_name}}(Mapping):
|
|||
"slots": "(" + (", ".join(quote(s) for s in slots)) + ")",
|
||||
"required": "{" + (", ".join(quote(s) for s in required)) + "}",
|
||||
"nulls": "{" + (", ".join(quote(s) for s in nulls)) + "}",
|
||||
"defaults": jx_expression({"literal": defaults}).to_python(),
|
||||
"defaults": Literal(defaults).to_python(),
|
||||
"len_slots": len(slots),
|
||||
"dict": "{" + (", ".join(quote(s) + ": self." + s for s in slots)) + "}",
|
||||
"assign": "; ".join("_set(output, "+quote(s)+", self."+s+")" for s in slots),
|
||||
"types": "{" + (",".join(quote(k) + ": " + v.__name__ for k, v in types.items())) + "}",
|
||||
"constraint_expr": jx_expression(constraint).to_python(),
|
||||
"constraint": value2json(constraint)
|
||||
}
|
||||
"assign": "; ".join(
|
||||
"_set(output, " + quote(s) + ", self." + s + ")" for s in slots
|
||||
),
|
||||
"types": "{"
|
||||
+ (",".join(quote(k) + ": " + v.__name__ for k, v in types.items()))
|
||||
+ "}",
|
||||
"constraint_expr": Python[jx_expression(not ENABLE_CONSTRAINTS or constraint)].to_python(),
|
||||
"constraint": value2json(constraint),
|
||||
},
|
||||
)
|
||||
|
||||
return _exec(code, name)
|
||||
output = _exec(code, name)
|
||||
register_data(output)
|
||||
return output
|
||||
|
||||
|
||||
class TableDesc(DataClass(
|
||||
TableDesc = DataClass(
|
||||
"Table",
|
||||
[
|
||||
"name",
|
||||
"url",
|
||||
"query_path",
|
||||
"timestamp"
|
||||
],
|
||||
constraint={"and": [
|
||||
{"eq": [{"last": "query_path"}, {"literal": "."}]}
|
||||
]}
|
||||
)):
|
||||
@property
|
||||
def columns(self):
|
||||
raise NotImplementedError()
|
||||
# return singlton.get_columns(table_name=self.name)
|
||||
["name", "url", "query_path", {"name": "last_updated", "nulls": False}, "columns"],
|
||||
constraint={"and": [{"eq": [{"last": "query_path"}, {"literal": "."}]}]},
|
||||
)
|
||||
|
||||
|
||||
Column = DataClass(
|
||||
"Column",
|
||||
[
|
||||
# "table",
|
||||
"names", # MAP FROM TABLE NAME TO COLUMN NAME (ONE COLUMN CAN HAVE MULTIPLE NAMES)
|
||||
"name",
|
||||
"es_column",
|
||||
"es_index",
|
||||
"es_type",
|
||||
{"name": "jx_type", "nulls": True},
|
||||
"jx_type",
|
||||
{"name": "useSource", "default": False},
|
||||
{"name": "nested_path", "nulls": True}, # AN ARRAY OF PATHS (FROM DEEPEST TO SHALLOWEST) INDICATING THE JSON SUB-ARRAYS
|
||||
"nested_path", # AN ARRAY OF PATHS (FROM DEEPEST TO SHALLOWEST) INDICATING THE JSON SUB-ARRAYS
|
||||
{"name": "count", "nulls": True},
|
||||
{"name": "cardinality", "nulls": True},
|
||||
{"name": "multi", "nulls": True},
|
||||
{"name": "multi", "nulls": False},
|
||||
{"name": "partitions", "nulls": True},
|
||||
{"name": "last_updated", "nulls": True}
|
||||
"last_updated",
|
||||
],
|
||||
constraint={"and": [
|
||||
{"eq": [{"last": "nested_path"}, {"literal": "."}]}
|
||||
]}
|
||||
constraint={
|
||||
"and": [
|
||||
{"not": {"find": {"es_column": "null"}}},
|
||||
{"not": {"eq": {"es_column": "string"}}},
|
||||
{"not": {"eq": {"es_type": "object", "jx_type": "exists"}}},
|
||||
{"eq": [{"last": "nested_path"}, {"literal": "."}]},
|
||||
{
|
||||
"when": {"eq": [{"literal": ".~N~"}, {"right": {"es_column": 4}}]},
|
||||
"then": {"gt": {"multi": 1}},
|
||||
"else": True,
|
||||
},
|
||||
{
|
||||
"when": {"gte": [{"count": "nested_path"}, 2]},
|
||||
"then": {"ne": [{"first": {"right": {"nested_path": 2}}}, {"literal": "."}]}, # SECOND-LAST ELEMENT
|
||||
"else": True
|
||||
}
|
||||
]
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
from jx_base.container import Container
|
||||
from jx_base.namespace import Namespace
|
||||
from jx_base.facts import Facts
|
||||
from jx_base.snowflake import Snowflake
|
||||
from jx_base.table import Table
|
||||
from jx_base.schema import Schema
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -5,18 +5,14 @@
|
|||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Author: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import unicode_literals
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from collections import Mapping
|
||||
from copy import copy
|
||||
|
||||
from mo_dots import Data
|
||||
from mo_dots import set_default, split_field, wrap, join_field
|
||||
from mo_future import generator_types, text_type
|
||||
from mo_dots import Data, is_data, is_many, join_field, set_default, split_field, wrap
|
||||
from mo_future import is_text
|
||||
from mo_logs import Log
|
||||
|
||||
type2container = Data()
|
||||
|
|
@ -28,7 +24,6 @@ _Query = None
|
|||
|
||||
|
||||
def _delayed_imports():
|
||||
global type2container
|
||||
global _ListContainer
|
||||
global _Cube
|
||||
global _run
|
||||
|
|
@ -47,9 +42,9 @@ def _delayed_imports():
|
|||
|
||||
class Container(object):
|
||||
"""
|
||||
CONTAINERS HOLD MULTIPLE FACTS AND CAN HANDLE
|
||||
CONTAINERS HOLD MULTIPLE INDICES AND CAN HANDLE
|
||||
GENERAL JSON QUERY EXPRESSIONS ON ITS CONTENTS
|
||||
METADATA FOR A Container IS CALL A Namespace
|
||||
METADATA FOR A Container IS CALLED A Namespace
|
||||
"""
|
||||
|
||||
|
||||
|
|
@ -67,9 +62,9 @@ class Container(object):
|
|||
return frum
|
||||
elif isinstance(frum, _Query):
|
||||
return _run(frum)
|
||||
elif isinstance(frum, (list, set) + generator_types):
|
||||
elif is_many(frum):
|
||||
return _ListContainer(frum)
|
||||
elif isinstance(frum, text_type):
|
||||
elif is_text(frum):
|
||||
# USE DEFAULT STORAGE TO FIND Container
|
||||
if not config.default.settings:
|
||||
Log.error("expecting jx_base.container.config.default.settings to contain default elasticsearch connection info")
|
||||
|
|
@ -83,7 +78,7 @@ class Container(object):
|
|||
)
|
||||
settings.type = None # WE DO NOT WANT TO INFLUENCE THE TYPE BECAUSE NONE IS IN THE frum STRING ANYWAY
|
||||
return type2container["elasticsearch"](settings)
|
||||
elif isinstance(frum, Mapping):
|
||||
elif is_data(frum):
|
||||
frum = wrap(frum)
|
||||
if frum.type and type2container[frum.type]:
|
||||
return type2container[frum.type](frum.settings)
|
||||
|
|
@ -119,10 +114,6 @@ class Container(object):
|
|||
def window(self, window):
|
||||
raise NotImplementedError()
|
||||
|
||||
def having(self, having):
|
||||
_ = having
|
||||
raise NotImplementedError()
|
||||
|
||||
def format(self, format):
|
||||
_ = format
|
||||
raise NotImplementedError()
|
||||
|
|
|
|||
|
|
@ -5,19 +5,14 @@
|
|||
# License, v. 2.0. If a copy of the MPL was not distributed with self file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Author: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from collections import Mapping
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.domains import ALGEBRAIC, Domain, KNOWN
|
||||
from mo_dots import Data, FlatList, Null, coalesce, is_data, is_list, join_field, listwrap, split_field, wrap
|
||||
import mo_dots as dot
|
||||
from jx_base.domains import Domain, ALGEBRAIC, KNOWN
|
||||
from mo_dots import Null, coalesce, join_field, split_field, Data
|
||||
from mo_dots import wrap, listwrap
|
||||
from mo_dots.lists import FlatList
|
||||
from mo_future import transpose
|
||||
from mo_logs import Log
|
||||
from mo_math import SUM
|
||||
from mo_times.timer import Timer
|
||||
|
|
@ -56,7 +51,7 @@ class Dimension(object):
|
|||
fields = coalesce(dim.field, dim.fields)
|
||||
if not fields:
|
||||
return # NO FIELDS TO SEARCH
|
||||
elif isinstance(fields, Mapping):
|
||||
elif is_data(fields):
|
||||
self.fields = wrap(fields)
|
||||
edges = wrap([{"name": k, "value": v, "allowNulls": False} for k, v in self.fields.items()])
|
||||
else:
|
||||
|
|
@ -88,7 +83,7 @@ class Dimension(object):
|
|||
temp = Data(partitions=[])
|
||||
for i, count in enumerate(parts):
|
||||
a = dim.path(d.getEnd(d.partitions[i]))
|
||||
if not isinstance(a, list):
|
||||
if not is_list(a):
|
||||
Log.error("The path function on " + dim.name + " must return an ARRAY of parts")
|
||||
addParts(
|
||||
temp,
|
||||
|
|
@ -98,7 +93,7 @@ class Dimension(object):
|
|||
)
|
||||
self.value = coalesce(dim.value, "name")
|
||||
self.partitions = temp.partitions
|
||||
elif isinstance(fields, Mapping):
|
||||
elif is_data(fields):
|
||||
self.value = "name" # USE THE "name" ATTRIBUTE OF PARTS
|
||||
|
||||
partitions = FlatList()
|
||||
|
|
@ -135,7 +130,7 @@ class Dimension(object):
|
|||
array = parts.data.values()[0].cube # DIG DEEP INTO RESULT (ASSUME SINGLE VALUE CUBE, WITH NULL AT END)
|
||||
|
||||
def edges2value(*values):
|
||||
if isinstance(fields, Mapping):
|
||||
if is_data(fields):
|
||||
output = Data()
|
||||
for e, v in transpose(edges, values):
|
||||
output[e.name] = v
|
||||
|
|
@ -192,7 +187,7 @@ class Dimension(object):
|
|||
def getDomain(self, **kwargs):
|
||||
# kwargs.depth IS MEANT TO REACH INTO SUB-PARTITIONS
|
||||
kwargs = wrap(kwargs)
|
||||
kwargs.depth = coalesce(kwargs.depth, len(self.fields)-1 if isinstance(self.fields, list) else None)
|
||||
kwargs.depth = coalesce(kwargs.depth, len(self.fields)-1 if is_list(self.fields) else None)
|
||||
|
||||
if not self.partitions and self.edges:
|
||||
# USE EACH EDGE AS A PARTITION, BUT isFacet==True SO IT ALLOWS THE OVERLAP
|
||||
|
|
|
|||
|
|
@ -5,23 +5,17 @@
|
|||
# License, v. 2.0. If a copy of the MPL was not distributed with self file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Author: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import unicode_literals
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
import itertools
|
||||
from collections import Mapping
|
||||
from numbers import Number
|
||||
|
||||
from mo_future import text_type
|
||||
|
||||
from jx_base.expressions import jx_expression
|
||||
from mo_collections.unique_index import UniqueIndex
|
||||
from mo_dots import coalesce, Data, set_default, Null, listwrap
|
||||
from mo_dots import wrap
|
||||
from mo_dots.lists import FlatList
|
||||
from mo_dots import Data, FlatList, Null, coalesce, is_container, is_data, listwrap, set_default, unwrap, wrap
|
||||
from mo_future import text
|
||||
from mo_logs import Log
|
||||
from mo_math import MAX, MIN
|
||||
from mo_times.dates import Date
|
||||
|
|
@ -210,7 +204,12 @@ class SimpleSetDomain(Domain):
|
|||
DOMAIN IS A LIST OF OBJECTS, EACH WITH A value PROPERTY
|
||||
"""
|
||||
|
||||
__slots__ = ["NULL", "partitions", "map", "order"]
|
||||
__slots__ = [
|
||||
"NULL", # THE value FOR NULL
|
||||
"partitions", # LIST OF {name, value, dataIndex} dicts
|
||||
"map", # MAP FROM value TO name
|
||||
"order" # MAP FROM value TO dataIndex
|
||||
]
|
||||
|
||||
def __init__(self, **desc):
|
||||
Domain.__init__(self, **desc)
|
||||
|
|
@ -225,7 +224,7 @@ class SimpleSetDomain(Domain):
|
|||
if isinstance(self.key, set):
|
||||
Log.error("problem")
|
||||
|
||||
if not desc.key and (len(desc.partitions)==0 or isinstance(desc.partitions[0], (text_type, Number, tuple))):
|
||||
if not desc.key and (len(desc.partitions)==0 or isinstance(desc.partitions[0], (text, Number, tuple))):
|
||||
# ASSUME PARTS ARE STRINGS, CONVERT TO REAL PART OBJECTS
|
||||
self.key = "value"
|
||||
self.map = {}
|
||||
|
|
@ -236,7 +235,7 @@ class SimpleSetDomain(Domain):
|
|||
self.map[p] = part
|
||||
self.order[p] = i
|
||||
if isinstance(p, (int, float)):
|
||||
text_part = text_type(float(p)) # ES CAN NOT HANDLE NUMERIC PARTS
|
||||
text_part = text(float(p)) # ES CAN NOT HANDLE NUMERIC PARTS
|
||||
self.map[text_part] = part
|
||||
self.order[text_part] = i
|
||||
self.label = coalesce(self.label, "name")
|
||||
|
|
@ -246,15 +245,18 @@ class SimpleSetDomain(Domain):
|
|||
if desc.partitions and desc.dimension.fields and len(desc.dimension.fields) > 1:
|
||||
self.key = desc.key
|
||||
self.map = UniqueIndex(keys=desc.dimension.fields)
|
||||
elif desc.partitions and isinstance(desc.key, (list, set)):
|
||||
elif desc.partitions and is_container(desc.key):
|
||||
# TODO: desc.key CAN BE MUCH LIKE A SELECT, WHICH UniqueIndex CAN NOT HANDLE
|
||||
self.key = desc.key
|
||||
self.map = UniqueIndex(keys=desc.key)
|
||||
elif desc.partitions and isinstance(desc.partitions[0][desc.key], Mapping):
|
||||
elif desc.partitions and is_data(desc.partitions[0][desc.key]):
|
||||
# LOOKS LIKE OBJECTS
|
||||
# sorted = desc.partitions[desc.key]
|
||||
|
||||
self.key = desc.key
|
||||
self.map = UniqueIndex(keys=desc.key)
|
||||
# self.key = UNION(set(d[desc.key].keys()) for d in desc.partitions)
|
||||
# self.map = UniqueIndex(keys=self.key)
|
||||
self.order = {p[self.key]: p.dataIndex for p in desc.partitions}
|
||||
self.partitions = desc.partitions
|
||||
elif len(desc.partitions) == 0:
|
||||
# CREATE AN EMPTY DOMAIN
|
||||
self.key = "value"
|
||||
|
|
@ -376,7 +378,7 @@ class SetDomain(Domain):
|
|||
if isinstance(self.key, set):
|
||||
Log.error("problem")
|
||||
|
||||
if isinstance(desc.partitions[0], (int, float, text_type)):
|
||||
if isinstance(desc.partitions[0], (int, float, text)):
|
||||
# ASSMUE PARTS ARE STRINGS, CONVERT TO REAL PART OBJECTS
|
||||
self.key = "value"
|
||||
self.order[None] = len(desc.partitions)
|
||||
|
|
@ -388,14 +390,14 @@ class SetDomain(Domain):
|
|||
elif desc.partitions and desc.dimension.fields and len(desc.dimension.fields) > 1:
|
||||
self.key = desc.key
|
||||
self.map = UniqueIndex(keys=desc.dimension.fields)
|
||||
elif desc.partitions and isinstance(desc.key, (list, set)):
|
||||
elif desc.partitions and is_container(desc.key):
|
||||
# TODO: desc.key CAN BE MUCH LIKE A SELECT, WHICH UniqueIndex CAN NOT HANDLE
|
||||
self.key = desc.key
|
||||
self.map = UniqueIndex(keys=desc.key)
|
||||
elif desc.partitions and isinstance(desc.partitions[0][desc.key], Mapping):
|
||||
elif desc.partitions and is_data(desc.partitions[0][desc.key]):
|
||||
self.key = desc.key
|
||||
self.map = UniqueIndex(keys=desc.key)
|
||||
# self.key = UNION(set(d[desc.key].keys()) for d in desc.partitions)
|
||||
# self.key = UNION(*set(d[desc.key].keys()) for d in desc.partitions)
|
||||
# self.map = UniqueIndex(keys=self.key)
|
||||
elif desc.key == None:
|
||||
Log.error("Domains must have keys")
|
||||
|
|
@ -663,7 +665,7 @@ class RangeDomain(Domain):
|
|||
if not self.key:
|
||||
Log.error("Must have a key value")
|
||||
|
||||
parts = list(listwrap(self.partitions))
|
||||
parts =listwrap(self.partitions)
|
||||
for i, p in enumerate(parts):
|
||||
self.min = MIN([self.min, p.min])
|
||||
self.max = MAX([self.max, p.max])
|
||||
|
|
@ -675,10 +677,10 @@ class RangeDomain(Domain):
|
|||
|
||||
# VERIFY PARTITIONS DO NOT OVERLAP, HOLES ARE FINE
|
||||
for p, q in itertools.product(parts, parts):
|
||||
if p is not q and p.min <= q.min and q.min < p.max:
|
||||
if p.min <= q.min and q.min < p.max and unwrap(p) is not unwrap(q):
|
||||
Log.error("partitions overlap!")
|
||||
|
||||
self.partitions = parts
|
||||
self.partitions = wrap(parts)
|
||||
return
|
||||
elif any([self.min == None, self.max == None, self.interval == None]):
|
||||
Log.error("Can not handle missing parameter")
|
||||
|
|
|
|||
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
|
|
@ -0,0 +1,169 @@
|
|||
from jx_base.expressions._utils import simplified, extend, jx_expression, merge_types, operators, language, _jx_expression
|
||||
from jx_base.expressions.abs_op import AbsOp
|
||||
from jx_base.expressions.add_op import AddOp
|
||||
from jx_base.expressions.and_op import AndOp
|
||||
from jx_base.expressions.base_binary_op import BaseBinaryOp
|
||||
from jx_base.expressions.base_inequality_op import BaseInequalityOp
|
||||
from jx_base.expressions.base_multi_op import BaseMultiOp
|
||||
from jx_base.expressions.basic_add_op import BasicAddOp
|
||||
from jx_base.expressions.basic_eq_op import BasicEqOp
|
||||
from jx_base.expressions.basic_index_of_op import BasicIndexOfOp
|
||||
from jx_base.expressions.basic_mul_op import BasicMulOp
|
||||
from jx_base.expressions.basic_multi_op import BasicMultiOp
|
||||
from jx_base.expressions.basic_starts_with_op import BasicStartsWithOp
|
||||
from jx_base.expressions.basic_substring_op import BasicSubstringOp
|
||||
from jx_base.expressions.between_op import BetweenOp
|
||||
from jx_base.expressions.boolean_op import BooleanOp
|
||||
from jx_base.expressions.case_op import CaseOp
|
||||
from jx_base.expressions.coalesce_op import CoalesceOp
|
||||
from jx_base.expressions.concat_op import ConcatOp
|
||||
from jx_base.expressions.count_op import CountOp
|
||||
from jx_base.expressions.date_op import DateOp
|
||||
from jx_base.expressions.div_op import DivOp
|
||||
from jx_base.expressions.eq_op import EqOp
|
||||
from jx_base.expressions.es_nested_op import EsNestedOp
|
||||
from jx_base.expressions.es_script import EsScript
|
||||
from jx_base.expressions.exists_op import ExistsOp
|
||||
from jx_base.expressions.exp_op import ExpOp
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.false_op import FalseOp, FALSE
|
||||
from jx_base.expressions.find_op import FindOp
|
||||
from jx_base.expressions.first_op import FirstOp
|
||||
from jx_base.expressions.floor_op import FloorOp
|
||||
from jx_base.expressions.from_unix_op import FromUnixOp
|
||||
from jx_base.expressions.get_op import GetOp
|
||||
from jx_base.expressions.gt_op import GtOp
|
||||
from jx_base.expressions.gte_op import GteOp
|
||||
from jx_base.expressions.in_op import InOp
|
||||
from jx_base.expressions.integer_op import IntegerOp
|
||||
from jx_base.expressions.is_boolean_op import IsBooleanOp
|
||||
from jx_base.expressions.is_integer_op import IsIntegerOp
|
||||
from jx_base.expressions.is_number_op import IsNumberOp
|
||||
from jx_base.expressions.is_string_op import IsStringOp
|
||||
from jx_base.expressions.last_op import LastOp
|
||||
from jx_base.expressions.leaves_op import LeavesOp
|
||||
from jx_base.expressions.left_op import LeftOp
|
||||
from jx_base.expressions.length_op import LengthOp
|
||||
from jx_base.expressions.literal import Literal, ONE, ZERO, register_literal, is_literal
|
||||
from jx_base.expressions.lt_op import LtOp
|
||||
from jx_base.expressions.lte_op import LteOp
|
||||
from jx_base.expressions.max_op import MaxOp
|
||||
from jx_base.expressions.min_op import MinOp
|
||||
from jx_base.expressions.missing_op import MissingOp
|
||||
from jx_base.expressions.mod_op import ModOp
|
||||
from jx_base.expressions.mul_op import MulOp
|
||||
from jx_base.expressions.ne_op import NeOp
|
||||
from jx_base.expressions.not_left_op import NotLeftOp
|
||||
from jx_base.expressions.not_op import NotOp
|
||||
from jx_base.expressions.not_right_op import NotRightOp
|
||||
from jx_base.expressions.null_op import NullOp, NULL
|
||||
from jx_base.expressions.number_op import NumberOp
|
||||
from jx_base.expressions.offset_op import OffsetOp
|
||||
from jx_base.expressions.or_op import OrOp
|
||||
from jx_base.expressions.prefix_op import PrefixOp
|
||||
from jx_base.expressions.python_script import PythonScript
|
||||
from jx_base.expressions.query_op import QueryOp
|
||||
from jx_base.expressions.range_op import RangeOp
|
||||
from jx_base.expressions.reg_exp_op import RegExpOp
|
||||
from jx_base.expressions.right_op import RightOp
|
||||
from jx_base.expressions.rows_op import RowsOp
|
||||
from jx_base.expressions.script_op import ScriptOp
|
||||
from jx_base.expressions.select_op import SelectOp
|
||||
from jx_base.expressions.split_op import SplitOp
|
||||
from jx_base.expressions.sql_eq_op import SqlEqOp
|
||||
from jx_base.expressions.sql_instr_op import SqlInstrOp
|
||||
from jx_base.expressions.sql_script import SQLScript
|
||||
from jx_base.expressions.sql_substr_op import SqlSubstrOp
|
||||
from jx_base.expressions.string_op import StringOp
|
||||
from jx_base.expressions.sub_op import SubOp
|
||||
from jx_base.expressions.suffix_op import SuffixOp
|
||||
from jx_base.expressions.true_op import TrueOp, TRUE
|
||||
from jx_base.expressions.tuple_op import TupleOp
|
||||
from jx_base.expressions.union_op import UnionOp
|
||||
from jx_base.expressions.unix_op import UnixOp
|
||||
from jx_base.expressions.variable import Variable, IDENTITY
|
||||
from jx_base.expressions.when_op import WhenOp
|
||||
from mo_dots import set_default
|
||||
|
||||
set_default(operators, {
|
||||
"abs": AbsOp,
|
||||
"add": AddOp,
|
||||
"and": AndOp,
|
||||
"basic.add": BasicAddOp,
|
||||
"basic.mul": BasicMulOp,
|
||||
"between": BetweenOp,
|
||||
"case": CaseOp,
|
||||
"coalesce": CoalesceOp,
|
||||
"concat": ConcatOp,
|
||||
"count": CountOp,
|
||||
"date": DateOp,
|
||||
"div": DivOp,
|
||||
"divide": DivOp,
|
||||
"eq": EqOp,
|
||||
"exists": ExistsOp,
|
||||
"exp": ExpOp,
|
||||
"find": FindOp,
|
||||
"first": FirstOp,
|
||||
"floor": FloorOp,
|
||||
"from_unix": FromUnixOp,
|
||||
"get": GetOp,
|
||||
"gt": GtOp,
|
||||
"gte": GteOp,
|
||||
"in": InOp,
|
||||
"instr": FindOp,
|
||||
"is_number": IsNumberOp,
|
||||
"is_string": IsStringOp,
|
||||
"last": LastOp,
|
||||
"left": LeftOp,
|
||||
"length": LengthOp,
|
||||
"literal": Literal,
|
||||
"lt": LtOp,
|
||||
"lte": LteOp,
|
||||
"match_all": TrueOp,
|
||||
"max": MaxOp,
|
||||
"minus": SubOp,
|
||||
"missing": MissingOp,
|
||||
"mod": ModOp,
|
||||
"mul": MulOp,
|
||||
"mult": MulOp,
|
||||
"multiply": MulOp,
|
||||
"ne": NeOp,
|
||||
"neq": NeOp,
|
||||
"not": NotOp,
|
||||
"not_left": NotLeftOp,
|
||||
"not_right": NotRightOp,
|
||||
"null": NullOp,
|
||||
"number": NumberOp,
|
||||
"offset": OffsetOp,
|
||||
"or": OrOp,
|
||||
"postfix": SuffixOp,
|
||||
"prefix": PrefixOp,
|
||||
"range": RangeOp,
|
||||
"regex": RegExpOp,
|
||||
"regexp": RegExpOp,
|
||||
"right": RightOp,
|
||||
"rows": RowsOp,
|
||||
"script": ScriptOp,
|
||||
"select": SelectOp,
|
||||
"split": SplitOp,
|
||||
"string": StringOp,
|
||||
"suffix": SuffixOp,
|
||||
"sub": SubOp,
|
||||
"subtract": SubOp,
|
||||
"sum": AddOp,
|
||||
"term": EqOp,
|
||||
"terms": InOp,
|
||||
"tuple": TupleOp,
|
||||
"union": UnionOp,
|
||||
"unix": UnixOp,
|
||||
"when": WhenOp,
|
||||
})
|
||||
|
||||
language.register_ops(vars())
|
||||
|
||||
register_literal(NullOp)
|
||||
register_literal(FalseOp)
|
||||
register_literal(TrueOp)
|
||||
register_literal(DateOp)
|
||||
register_literal(Literal)
|
||||
|
||||
|
|
@ -0,0 +1,180 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
import operator
|
||||
|
||||
from jx_base.language import is_expression, Language
|
||||
from mo_dots import Null, is_sequence
|
||||
from mo_future import (
|
||||
first,
|
||||
get_function_name,
|
||||
is_text,
|
||||
items as items_,
|
||||
text,
|
||||
utf8_json_encoder,
|
||||
)
|
||||
from mo_json import BOOLEAN, INTEGER, IS_NULL, NUMBER, OBJECT, STRING, scrub
|
||||
from mo_logs import Except, Log
|
||||
from mo_math import is_number
|
||||
from mo_times import Date
|
||||
|
||||
ALLOW_SCRIPTING = False
|
||||
EMPTY_DICT = {}
|
||||
|
||||
Literal, TRUE, NULL, TupleOp, Variable = [None] * 5
|
||||
|
||||
def extend(cls):
|
||||
"""
|
||||
DECORATOR TO ADD METHODS TO CLASSES
|
||||
:param cls: THE CLASS TO ADD THE METHOD TO
|
||||
:return:
|
||||
"""
|
||||
|
||||
def extender(func):
|
||||
setattr(cls, get_function_name(func), func)
|
||||
return func
|
||||
|
||||
return extender
|
||||
|
||||
|
||||
def simplified(func):
|
||||
def mark_as_simple(self):
|
||||
if self.simplified:
|
||||
return self
|
||||
|
||||
output = func(self)
|
||||
output.simplified = True
|
||||
return output
|
||||
|
||||
func_name = get_function_name(func)
|
||||
mark_as_simple.__name__ = func_name
|
||||
return mark_as_simple
|
||||
|
||||
|
||||
def jx_expression(expr, schema=None):
|
||||
if expr == None:
|
||||
return None
|
||||
|
||||
# UPDATE THE VARIABLE WITH THIER KNOWN TYPES
|
||||
if not schema:
|
||||
output = _jx_expression(expr, language)
|
||||
return output
|
||||
output = _jx_expression(expr, language)
|
||||
for v in output.vars():
|
||||
leaves = schema.leaves(v.var)
|
||||
if len(leaves) == 0:
|
||||
v.data_type = IS_NULL
|
||||
if len(leaves) == 1:
|
||||
v.data_type = first(leaves).jx_type
|
||||
return output
|
||||
|
||||
|
||||
def _jx_expression(expr, lang):
|
||||
"""
|
||||
WRAP A JSON EXPRESSION WITH OBJECT REPRESENTATION
|
||||
"""
|
||||
if is_expression(expr):
|
||||
# CONVERT TO lang
|
||||
new_op = lang[expr]
|
||||
if not new_op:
|
||||
# CAN NOT BE FOUND, TRY SOME PARTIAL EVAL
|
||||
return language[expr.get_id()].partial_eval()
|
||||
return expr
|
||||
# return new_op(expr.args) # THIS CAN BE DONE, BUT IT NEEDS MORE CODING, AND I WOULD EXPECT IT TO BE SLOW
|
||||
|
||||
if expr is None:
|
||||
return TRUE
|
||||
elif is_text(expr):
|
||||
return Variable(expr)
|
||||
elif expr in (True, False, None) or expr == None or is_number(expr):
|
||||
return Literal(expr)
|
||||
elif expr.__class__ is Date:
|
||||
return Literal(expr.unix)
|
||||
elif is_sequence(expr):
|
||||
return lang[TupleOp([_jx_expression(e, lang) for e in expr])]
|
||||
|
||||
# expr = wrap(expr)
|
||||
try:
|
||||
items = items_(expr)
|
||||
|
||||
for op, term in items:
|
||||
# ONE OF THESE IS THE OPERATOR
|
||||
full_op = operators.get(op)
|
||||
if full_op:
|
||||
class_ = lang.ops[full_op.get_id()]
|
||||
if class_:
|
||||
return class_.define(expr)
|
||||
|
||||
# THIS LANGUAGE DOES NOT SUPPORT THIS OPERATOR, GOTO BASE LANGUAGE AND GET THE MACRO
|
||||
class_ = language[op.get_id()]
|
||||
output = class_.define(expr).partial_eval()
|
||||
return _jx_expression(output, lang)
|
||||
else:
|
||||
if not items:
|
||||
return NULL
|
||||
raise Log.error("{{instruction|json}} is not known", instruction=expr)
|
||||
|
||||
except Exception as e:
|
||||
Log.error("programmer error expr = {{value|quote}}", value=expr, cause=e)
|
||||
|
||||
|
||||
language = Language(None)
|
||||
|
||||
|
||||
_json_encoder = utf8_json_encoder
|
||||
|
||||
|
||||
def value2json(value):
|
||||
try:
|
||||
scrubbed = scrub(value, scrub_number=float)
|
||||
return text(_json_encoder(scrubbed))
|
||||
except Exception as e:
|
||||
e = Except.wrap(e)
|
||||
Log.warning("problem serializing {{type}}", type=text(repr(value)), cause=e)
|
||||
raise e
|
||||
|
||||
|
||||
def merge_types(jx_types):
|
||||
"""
|
||||
:param jx_types: ITERABLE OF jx TYPES
|
||||
:return: ONE TYPE TO RULE THEM ALL
|
||||
"""
|
||||
return _merge_types[max(_merge_score[t] for t in jx_types)]
|
||||
|
||||
|
||||
_merge_score = {IS_NULL: 0, BOOLEAN: 1, INTEGER: 2, NUMBER: 3, STRING: 4, OBJECT: 5}
|
||||
_merge_types = {v: k for k, v in _merge_score.items()}
|
||||
|
||||
builtin_ops = {
|
||||
"ne": operator.ne,
|
||||
"eq": operator.eq,
|
||||
"gte": operator.ge,
|
||||
"gt": operator.gt,
|
||||
"lte": operator.le,
|
||||
"lt": operator.lt,
|
||||
"add": operator.add,
|
||||
"sub": operator.sub,
|
||||
"mul": operator.mul,
|
||||
"max": lambda *v: max(v),
|
||||
"min": lambda *v: min(v),
|
||||
}
|
||||
|
||||
operators = {}
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions._utils import simplified
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.language import is_op
|
||||
from mo_json import NUMBER
|
||||
|
||||
|
||||
class AbsOp(Expression):
|
||||
data_type = NUMBER
|
||||
|
||||
def __init__(self, term):
|
||||
Expression.__init__(self, term)
|
||||
self.term = term
|
||||
|
||||
def __data__(self):
|
||||
return {"abs": self.term.__data__()}
|
||||
|
||||
def __eq__(self, other):
|
||||
if not is_op(other, AbsOp):
|
||||
return False
|
||||
return self.term == other.term
|
||||
|
||||
def vars(self):
|
||||
return self.term.vars()
|
||||
|
||||
def map(self, map_):
|
||||
return self.lang[AbsOp(self.term.map(map_))]
|
||||
|
||||
def missing(self):
|
||||
return self.term.missing()
|
||||
|
||||
@simplified
|
||||
def partial_eval(self):
|
||||
return AbsOp(self.term.partial_eval())
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions.base_multi_op import BaseMultiOp
|
||||
|
||||
|
||||
class AddOp(BaseMultiOp):
|
||||
op = "add"
|
||||
|
|
@ -0,0 +1,115 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions._utils import simplified
|
||||
from jx_base.expressions.boolean_op import BooleanOp
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.false_op import FALSE
|
||||
from jx_base.expressions.true_op import TRUE
|
||||
from jx_base.language import is_op
|
||||
from mo_dots import is_many
|
||||
from mo_future import zip_longest
|
||||
from mo_json import BOOLEAN
|
||||
|
||||
NotOp = None
|
||||
OrOp = None
|
||||
|
||||
class AndOp(Expression):
|
||||
data_type = BOOLEAN
|
||||
|
||||
def __init__(self, terms):
|
||||
Expression.__init__(self, terms)
|
||||
if terms == None:
|
||||
self.terms = []
|
||||
elif is_many(terms):
|
||||
self.terms = terms
|
||||
else:
|
||||
self.terms = [terms]
|
||||
|
||||
def __data__(self):
|
||||
return {"and": [t.__data__() for t in self.terms]}
|
||||
|
||||
def __eq__(self, other):
|
||||
if is_op(other, AndOp):
|
||||
return all(a == b for a, b in zip_longest(self.terms, other.terms))
|
||||
return False
|
||||
|
||||
def vars(self):
|
||||
output = set()
|
||||
for t in self.terms:
|
||||
output |= t.vars()
|
||||
return output
|
||||
|
||||
def map(self, map_):
|
||||
return self.lang[AndOp([t.map(map_) for t in self.terms])]
|
||||
|
||||
def missing(self):
|
||||
return FALSE
|
||||
|
||||
@simplified
|
||||
def partial_eval(self):
|
||||
or_terms = [[]] # LIST OF TUPLES FOR or-ing and and-ing
|
||||
for i, t in enumerate(self.terms):
|
||||
simple = self.lang[BooleanOp(t)].partial_eval()
|
||||
if simple.type != BOOLEAN:
|
||||
simple = simple.exists()
|
||||
|
||||
if simple is self.lang[TRUE]:
|
||||
continue
|
||||
elif simple is FALSE:
|
||||
return FALSE
|
||||
elif is_op(simple, AndOp):
|
||||
for and_terms in or_terms:
|
||||
and_terms.extend([tt for tt in simple.terms if tt not in and_terms])
|
||||
continue
|
||||
elif is_op(simple, OrOp):
|
||||
or_terms = [
|
||||
and_terms + ([o] if o not in and_terms else [])
|
||||
for o in simple.terms
|
||||
for and_terms in or_terms
|
||||
if self.lang[NotOp(o)].partial_eval() not in and_terms
|
||||
]
|
||||
continue
|
||||
for and_terms in list(or_terms):
|
||||
if self.lang[NotOp(simple)].partial_eval() in and_terms:
|
||||
or_terms.remove(and_terms)
|
||||
elif simple not in and_terms:
|
||||
and_terms.append(simple)
|
||||
|
||||
if len(or_terms) == 0:
|
||||
return FALSE
|
||||
elif len(or_terms) == 1:
|
||||
and_terms = or_terms[0]
|
||||
if len(and_terms) == 0:
|
||||
return TRUE
|
||||
elif len(and_terms) == 1:
|
||||
return and_terms[0]
|
||||
else:
|
||||
return self.lang[AndOp(and_terms)]
|
||||
|
||||
return self.lang[
|
||||
OrOp(
|
||||
[
|
||||
AndOp(and_terms) if len(and_terms) > 1 else and_terms[0]
|
||||
for and_terms in or_terms
|
||||
]
|
||||
)
|
||||
].partial_eval()
|
||||
|
|
@ -0,0 +1,78 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions._utils import builtin_ops, simplified
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.false_op import FALSE
|
||||
from jx_base.expressions.literal import Literal
|
||||
from jx_base.expressions.literal import is_literal
|
||||
from jx_base.expressions.null_op import NULL
|
||||
from jx_base.expressions.or_op import OrOp
|
||||
from jx_base.expressions.variable import Variable
|
||||
from jx_base.language import is_op
|
||||
from mo_json import NUMBER
|
||||
|
||||
|
||||
class BaseBinaryOp(Expression):
|
||||
has_simple_form = True
|
||||
data_type = NUMBER
|
||||
op = None
|
||||
|
||||
def __init__(self, terms, default=NULL):
|
||||
Expression.__init__(self, terms)
|
||||
self.lhs, self.rhs = terms
|
||||
self.default = default
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
return self.op
|
||||
|
||||
def __data__(self):
|
||||
if is_op(self.lhs, Variable) and is_literal(self.rhs):
|
||||
return {self.op: {self.lhs.var, self.rhs.value}, "default": self.default}
|
||||
else:
|
||||
return {
|
||||
self.op: [self.lhs.__data__(), self.rhs.__data__()],
|
||||
"default": self.default,
|
||||
}
|
||||
|
||||
def vars(self):
|
||||
return self.lhs.vars() | self.rhs.vars() | self.default.vars()
|
||||
|
||||
def map(self, map_):
|
||||
return self.__class__(
|
||||
[self.lhs.map(map_), self.rhs.map(map_)], default=self.default.map(map_)
|
||||
)
|
||||
|
||||
def missing(self):
|
||||
if self.default.exists():
|
||||
return FALSE
|
||||
else:
|
||||
return self.lang[OrOp([self.lhs.missing(), self.rhs.missing()])]
|
||||
|
||||
@simplified
|
||||
def partial_eval(self):
|
||||
lhs = self.lhs.partial_eval()
|
||||
rhs = self.rhs.partial_eval()
|
||||
default = self.default.partial_eval()
|
||||
if is_literal(lhs) and is_literal(rhs):
|
||||
return Literal(builtin_ops[self.op](lhs.value, rhs.value))
|
||||
return self.__class__([lhs, rhs], default=default)
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions._utils import builtin_ops, simplified
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.false_op import FALSE
|
||||
from jx_base.expressions.literal import Literal
|
||||
from jx_base.expressions.literal import is_literal
|
||||
from jx_base.expressions.variable import Variable
|
||||
from jx_base.language import is_op
|
||||
from mo_json import BOOLEAN
|
||||
|
||||
|
||||
class BaseInequalityOp(Expression):
|
||||
has_simple_form = True
|
||||
data_type = BOOLEAN
|
||||
op = None
|
||||
|
||||
def __init__(self, terms):
|
||||
Expression.__init__(self, terms)
|
||||
self.lhs, self.rhs = terms
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
return self.op
|
||||
|
||||
def __data__(self):
|
||||
if is_op(self.lhs, Variable) and is_literal(self.rhs):
|
||||
return {self.op: {self.lhs.var, self.rhs.value}}
|
||||
else:
|
||||
return {self.op: [self.lhs.__data__(), self.rhs.__data__()]}
|
||||
|
||||
def __eq__(self, other):
|
||||
if not isinstance(other, self.__class__):
|
||||
return False
|
||||
return self.op == other.op and self.lhs == other.lhs and self.rhs == other.rhs
|
||||
|
||||
def vars(self):
|
||||
return self.lhs.vars() | self.rhs.vars()
|
||||
|
||||
def map(self, map_):
|
||||
return self.__class__([self.lhs.map(map_), self.rhs.map(map_)])
|
||||
|
||||
def missing(self):
|
||||
return FALSE
|
||||
|
||||
@simplified
|
||||
def partial_eval(self):
|
||||
lhs = self.lhs.partial_eval()
|
||||
rhs = self.rhs.partial_eval()
|
||||
|
||||
if is_literal(lhs) and is_literal(rhs):
|
||||
return Literal(builtin_ops[self.op](lhs, rhs))
|
||||
|
||||
return self.__class__([lhs, rhs])
|
||||
|
|
@ -0,0 +1,142 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions._utils import simplified, builtin_ops, operators
|
||||
from jx_base.expressions.and_op import AndOp
|
||||
from jx_base.expressions.coalesce_op import CoalesceOp
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.false_op import FALSE
|
||||
from jx_base.expressions.literal import Literal, ZERO, ONE, is_literal
|
||||
from jx_base.expressions.null_op import NULL
|
||||
from jx_base.expressions.or_op import OrOp
|
||||
from jx_base.expressions.true_op import TRUE
|
||||
from jx_base.expressions.when_op import WhenOp
|
||||
from mo_dots import coalesce
|
||||
from mo_json import NUMBER
|
||||
|
||||
|
||||
class BaseMultiOp(Expression):
|
||||
has_simple_form = True
|
||||
data_type = NUMBER
|
||||
op = None
|
||||
|
||||
def __init__(self, terms, **clauses):
|
||||
Expression.__init__(self, terms)
|
||||
self.terms = terms
|
||||
self.default = coalesce(clauses.get("default"), NULL)
|
||||
self.nulls = coalesce(
|
||||
clauses.get("nulls"), FALSE
|
||||
) # nulls==True WILL HAVE OP RETURN null ONLY IF ALL OPERANDS ARE null
|
||||
|
||||
def __data__(self):
|
||||
return {
|
||||
self.op: [t.__data__() for t in self.terms],
|
||||
"default": self.default,
|
||||
"nulls": self.nulls,
|
||||
}
|
||||
|
||||
def vars(self):
|
||||
output = set()
|
||||
for t in self.terms:
|
||||
output |= t.vars()
|
||||
return output
|
||||
|
||||
def map(self, map_):
|
||||
return self.__class__(
|
||||
[t.map(map_) for t in self.terms],
|
||||
**{"default": self.default, "nulls": self.nulls}
|
||||
)
|
||||
|
||||
def missing(self):
|
||||
if self.nulls:
|
||||
if self.default is NULL:
|
||||
return self.lang[AndOp([t.missing() for t in self.terms])]
|
||||
else:
|
||||
return TRUE
|
||||
else:
|
||||
if self.default is NULL:
|
||||
return self.lang[OrOp([t.missing() for t in self.terms])]
|
||||
else:
|
||||
return FALSE
|
||||
|
||||
def exists(self):
|
||||
if self.nulls:
|
||||
return self.lang[OrOp([t.exists() for t in self.terms])]
|
||||
else:
|
||||
return self.lang[AndOp([t.exists() for t in self.terms])]
|
||||
|
||||
@simplified
|
||||
def partial_eval(self):
|
||||
acc = None
|
||||
terms = []
|
||||
for t in self.terms:
|
||||
simple = t.partial_eval()
|
||||
if simple is NULL:
|
||||
pass
|
||||
elif is_literal(simple):
|
||||
if acc is None:
|
||||
acc = simple.value
|
||||
else:
|
||||
acc = builtin_ops[self.op](acc, simple.value)
|
||||
else:
|
||||
terms.append(simple)
|
||||
|
||||
lang = self.lang
|
||||
if len(terms) == 0:
|
||||
if acc == None:
|
||||
return self.default.partial_eval()
|
||||
else:
|
||||
return lang[Literal(acc)]
|
||||
elif self.nulls:
|
||||
# DECISIVE
|
||||
if acc is not None:
|
||||
terms.append(Literal(acc))
|
||||
|
||||
output = lang[
|
||||
WhenOp(
|
||||
AndOp([t.missing() for t in terms]),
|
||||
**{
|
||||
"then": self.default,
|
||||
"else": operators["basic." + self.op](
|
||||
[CoalesceOp([t, _jx_identity[self.op]]) for t in terms]
|
||||
),
|
||||
}
|
||||
)
|
||||
].partial_eval()
|
||||
else:
|
||||
# CONSERVATIVE
|
||||
if acc is not None:
|
||||
terms.append(lang[Literal(acc)])
|
||||
|
||||
output = lang[
|
||||
WhenOp(
|
||||
lang[OrOp([t.missing() for t in terms])],
|
||||
**{
|
||||
"then": self.default,
|
||||
"else": operators["basic." + self.op](terms),
|
||||
}
|
||||
)
|
||||
].partial_eval()
|
||||
|
||||
return output
|
||||
|
||||
|
||||
_jx_identity = {"add": ZERO, "mul": ONE}
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions.basic_multi_op import BasicMultiOp
|
||||
|
||||
|
||||
class BasicAddOp(BasicMultiOp):
|
||||
op = "basic.add"
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.false_op import FALSE
|
||||
from jx_base.language import is_op
|
||||
from mo_json import BOOLEAN
|
||||
|
||||
|
||||
class BasicEqOp(Expression):
|
||||
"""
|
||||
PLACEHOLDER FOR BASIC `==` OPERATOR (CAN NOT DEAL WITH NULLS)
|
||||
"""
|
||||
|
||||
data_type = BOOLEAN
|
||||
|
||||
def __init__(self, terms):
|
||||
Expression.__init__(self, terms)
|
||||
self.lhs, self.rhs = terms
|
||||
|
||||
def __data__(self):
|
||||
return {"basic.eq": [self.lhs.__data__(), self.rhs.__data__()]}
|
||||
|
||||
def missing(self):
|
||||
return FALSE
|
||||
|
||||
def __eq__(self, other):
|
||||
if not is_op(other, BasicEqOp):
|
||||
return False
|
||||
return self.lhs == other.lhs and self.rhs == other.rhs
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.language import is_op
|
||||
|
||||
from jx_base.expressions._utils import simplified
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.false_op import FALSE
|
||||
from jx_base.expressions.integer_op import IntegerOp
|
||||
from jx_base.expressions.literal import ZERO
|
||||
from jx_base.expressions.max_op import MaxOp
|
||||
from jx_base.expressions.string_op import StringOp
|
||||
from mo_json import INTEGER
|
||||
|
||||
|
||||
class BasicIndexOfOp(Expression):
|
||||
"""
|
||||
PLACEHOLDER FOR BASIC value.indexOf(find, start) (CAN NOT DEAL WITH NULLS)
|
||||
"""
|
||||
|
||||
data_type = INTEGER
|
||||
|
||||
def __init__(self, params):
|
||||
Expression.__init__(self, params)
|
||||
self.value, self.find, self.start = params
|
||||
|
||||
def __data__(self):
|
||||
return {
|
||||
"basic.indexOf": [
|
||||
self.value.__data__(),
|
||||
self.find.__data__(),
|
||||
self.start.__data__(),
|
||||
]
|
||||
}
|
||||
|
||||
def vars(self):
|
||||
return self.value.vars() | self.find.vars() | self.start.vars()
|
||||
|
||||
def missing(self):
|
||||
return FALSE
|
||||
|
||||
@simplified
|
||||
def partial_eval(self):
|
||||
start = IntegerOp(MaxOp([ZERO, self.start])).partial_eval()
|
||||
return self.lang[
|
||||
BasicIndexOfOp(
|
||||
[
|
||||
StringOp(self.value).partial_eval(),
|
||||
StringOp(self.find).partial_eval(),
|
||||
start,
|
||||
]
|
||||
)
|
||||
]
|
||||
|
||||
def __eq__(self, other):
|
||||
if not is_op(other, BasicIndexOfOp):
|
||||
return False
|
||||
return self.value == self.value and self.find == other.find and self.start == other.start
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions.basic_multi_op import BasicMultiOp
|
||||
|
||||
|
||||
class BasicMulOp(BasicMultiOp):
|
||||
op = "basic.mul"
|
||||
|
|
@ -0,0 +1,82 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions._utils import simplified, builtin_ops
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.false_op import FALSE
|
||||
from jx_base.expressions.literal import Literal
|
||||
from jx_base.expressions.null_op import NULL
|
||||
from jx_base.language import is_op
|
||||
from mo_json import NUMBER
|
||||
|
||||
|
||||
class BasicMultiOp(Expression):
|
||||
"""
|
||||
PLACEHOLDER FOR BASIC OPERATOR (CAN NOT DEAL WITH NULLS)
|
||||
"""
|
||||
|
||||
data_type = NUMBER
|
||||
op = None
|
||||
|
||||
def __init__(self, terms):
|
||||
Expression.__init__(self, terms)
|
||||
self.terms = terms
|
||||
|
||||
def vars(self):
|
||||
output = set()
|
||||
for t in self.terms:
|
||||
output.update(t.vars())
|
||||
return output
|
||||
|
||||
def map(self, map):
|
||||
return self.__class__([t.map(map) for t in self.terms])
|
||||
|
||||
def __data__(self):
|
||||
return {self.op: [t.__data__() for t in self.terms]}
|
||||
|
||||
def missing(self):
|
||||
return FALSE
|
||||
|
||||
@simplified
|
||||
def partial_eval(self):
|
||||
acc = None
|
||||
terms = []
|
||||
for t in self.terms:
|
||||
simple = t.partial_eval()
|
||||
if simple is NULL:
|
||||
pass
|
||||
elif is_op(simple, Literal):
|
||||
if acc is None:
|
||||
acc = simple.value
|
||||
else:
|
||||
acc = builtin_ops[self.op](acc, simple.value)
|
||||
else:
|
||||
terms.append(simple)
|
||||
if len(terms) == 0:
|
||||
if acc == None:
|
||||
return self.default.partial_eval()
|
||||
else:
|
||||
return Literal(acc)
|
||||
else:
|
||||
if acc is not None:
|
||||
terms.append(Literal(acc))
|
||||
|
||||
return self.__class__(terms)
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions._utils import simplified
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.false_op import FALSE
|
||||
from jx_base.expressions.string_op import StringOp
|
||||
from jx_base.language import is_op
|
||||
from mo_json import BOOLEAN
|
||||
|
||||
|
||||
class BasicStartsWithOp(Expression):
|
||||
"""
|
||||
PLACEHOLDER FOR BASIC value.startsWith(find, start) (CAN NOT DEAL WITH NULLS)
|
||||
"""
|
||||
|
||||
data_type = BOOLEAN
|
||||
|
||||
def __init__(self, params):
|
||||
Expression.__init__(self, params)
|
||||
self.value, self.prefix = params
|
||||
|
||||
def __data__(self):
|
||||
return {"basic.startsWith": [self.value.__data__(), self.prefix.__data__()]}
|
||||
|
||||
def __eq__(self, other):
|
||||
if is_op(other, BasicStartsWithOp):
|
||||
return self.value == other.value and self.prefix == other.prefix
|
||||
|
||||
def vars(self):
|
||||
return self.value.vars() | self.prefix.vars()
|
||||
|
||||
def missing(self):
|
||||
return FALSE
|
||||
|
||||
@simplified
|
||||
def partial_eval(self):
|
||||
return self.lang[
|
||||
BasicStartsWithOp(
|
||||
[
|
||||
StringOp(self.value).partial_eval(),
|
||||
StringOp(self.prefix).partial_eval(),
|
||||
]
|
||||
)
|
||||
]
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.false_op import FALSE
|
||||
from mo_json import STRING
|
||||
|
||||
|
||||
class BasicSubstringOp(Expression):
|
||||
"""
|
||||
PLACEHOLDER FOR BASIC value.substring(start, end) (CAN NOT DEAL WITH NULLS)
|
||||
"""
|
||||
|
||||
data_type = STRING
|
||||
|
||||
def __init__(self, terms):
|
||||
Expression.__init__(self, terms)
|
||||
self.value, self.start, self.end = terms
|
||||
|
||||
def __data__(self):
|
||||
return {
|
||||
"basic.substring": [
|
||||
self.value.__data__(),
|
||||
self.start.__data__(),
|
||||
self.end.__data__(),
|
||||
]
|
||||
}
|
||||
|
||||
def missing(self):
|
||||
return FALSE
|
||||
|
|
@ -0,0 +1,183 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions._utils import jx_expression, simplified
|
||||
from jx_base.expressions.add_op import AddOp
|
||||
from jx_base.expressions.basic_substring_op import BasicSubstringOp
|
||||
from jx_base.expressions.case_op import CaseOp
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.find_op import FindOp
|
||||
from jx_base.expressions.is_number_op import IsNumberOp
|
||||
from jx_base.expressions.length_op import LengthOp
|
||||
from jx_base.expressions.literal import Literal, ZERO, is_literal
|
||||
from jx_base.expressions.max_op import MaxOp
|
||||
from jx_base.expressions.min_op import MinOp
|
||||
from jx_base.expressions.null_op import NULL
|
||||
from jx_base.expressions.variable import Variable
|
||||
from jx_base.expressions.when_op import WhenOp
|
||||
from jx_base.language import is_op
|
||||
from mo_dots import is_data, is_sequence, wrap, coalesce
|
||||
from mo_json import STRING
|
||||
from mo_logs import Log
|
||||
|
||||
|
||||
class BetweenOp(Expression):
|
||||
data_type = STRING
|
||||
|
||||
def __init__(self, value, prefix, suffix, default=NULL, start=NULL):
|
||||
Expression.__init__(self, [])
|
||||
self.value = value
|
||||
self.prefix = coalesce(prefix, NULL)
|
||||
self.suffix = coalesce(suffix, NULL)
|
||||
self.default = coalesce(default, NULL)
|
||||
self.start = coalesce(start, NULL)
|
||||
if is_literal(self.prefix) and is_literal(self.suffix):
|
||||
pass
|
||||
else:
|
||||
Log.error("Expecting literal prefix and suffix only")
|
||||
|
||||
@classmethod
|
||||
def define(cls, expr):
|
||||
term = expr.between
|
||||
if is_sequence(term):
|
||||
return cls.lang[
|
||||
BetweenOp(
|
||||
value=jx_expression(term[0]),
|
||||
prefix=jx_expression(term[1]),
|
||||
suffix=jx_expression(term[2]),
|
||||
default=jx_expression(expr.default),
|
||||
start=jx_expression(expr.start),
|
||||
)
|
||||
]
|
||||
elif is_data(term):
|
||||
var, vals = term.items()[0]
|
||||
if is_sequence(vals) and len(vals) == 2:
|
||||
return cls.lang[
|
||||
BetweenOp(
|
||||
value=Variable(var),
|
||||
prefix=Literal(vals[0]),
|
||||
suffix=Literal(vals[1]),
|
||||
default=jx_expression(expr.default),
|
||||
start=jx_expression(expr.start),
|
||||
)
|
||||
]
|
||||
else:
|
||||
Log.error(
|
||||
"`between` parameters are expected to be in {var: [prefix, suffix]} form"
|
||||
)
|
||||
else:
|
||||
Log.error(
|
||||
"`between` parameters are expected to be in {var: [prefix, suffix]} form"
|
||||
)
|
||||
|
||||
def vars(self):
|
||||
return (
|
||||
self.value.vars()
|
||||
| self.prefix.vars()
|
||||
| self.suffix.vars()
|
||||
| self.default.vars()
|
||||
| self.start.vars()
|
||||
)
|
||||
|
||||
def map(self, map_):
|
||||
return BetweenOp(
|
||||
self.value.map(map_),
|
||||
self.prefix.map(map_),
|
||||
self.suffix.map(map_),
|
||||
default=self.default.map(map_),
|
||||
start=self.start.map(map_),
|
||||
)
|
||||
|
||||
def __data__(self):
|
||||
if (
|
||||
is_op(self.value, Variable)
|
||||
and is_literal(self.prefix)
|
||||
and is_literal(self.suffix)
|
||||
):
|
||||
output = wrap(
|
||||
{"between": {self.value.var: [self.prefix.value, self.suffix.value]}}
|
||||
)
|
||||
else:
|
||||
output = wrap(
|
||||
{
|
||||
"between": [
|
||||
self.value.__data__(),
|
||||
self.prefix.__data__(),
|
||||
self.suffix.__data__(),
|
||||
]
|
||||
}
|
||||
)
|
||||
if self.start:
|
||||
output.start = self.start.__data__()
|
||||
if self.default:
|
||||
output.default = self.default.__data__()
|
||||
return output
|
||||
|
||||
@simplified
|
||||
def partial_eval(self):
|
||||
value = self.value.partial_eval()
|
||||
|
||||
start_index = self.lang[
|
||||
CaseOp(
|
||||
[
|
||||
WhenOp(self.prefix.missing(), **{"then": ZERO}),
|
||||
WhenOp(
|
||||
IsNumberOp(self.prefix), **{"then": MaxOp([ZERO, self.prefix])}
|
||||
),
|
||||
FindOp([value, self.prefix], start=self.start),
|
||||
]
|
||||
)
|
||||
].partial_eval()
|
||||
|
||||
len_prefix = self.lang[
|
||||
CaseOp(
|
||||
[
|
||||
WhenOp(self.prefix.missing(), **{"then": ZERO}),
|
||||
WhenOp(IsNumberOp(self.prefix), **{"then": ZERO}),
|
||||
LengthOp(self.prefix),
|
||||
]
|
||||
)
|
||||
].partial_eval()
|
||||
|
||||
end_index = self.lang[
|
||||
CaseOp(
|
||||
[
|
||||
WhenOp(start_index.missing(), **{"then": NULL}),
|
||||
WhenOp(self.suffix.missing(), **{"then": LengthOp(value)}),
|
||||
WhenOp(
|
||||
IsNumberOp(self.suffix),
|
||||
**{"then": MinOp([self.suffix, LengthOp(value)])}
|
||||
),
|
||||
FindOp(
|
||||
[value, self.suffix], start=AddOp([start_index, len_prefix])
|
||||
),
|
||||
]
|
||||
)
|
||||
].partial_eval()
|
||||
|
||||
start_index = AddOp([start_index, len_prefix]).partial_eval()
|
||||
substring = BasicSubstringOp([value, start_index, end_index]).partial_eval()
|
||||
|
||||
between = self.lang[
|
||||
WhenOp(end_index.missing(), **{"then": self.default, "else": substring})
|
||||
].partial_eval()
|
||||
|
||||
return between
|
||||
|
|
@ -0,0 +1,62 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions._utils import simplified
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.false_op import FALSE
|
||||
from jx_base.expressions.null_op import NULL
|
||||
from jx_base.expressions.true_op import TRUE
|
||||
from mo_json import BOOLEAN
|
||||
|
||||
|
||||
class BooleanOp(Expression):
|
||||
data_type = BOOLEAN
|
||||
|
||||
def __init__(self, term):
|
||||
Expression.__init__(self, [term])
|
||||
self.term = term
|
||||
|
||||
def __data__(self):
|
||||
return {"boolean": self.term.__data__()}
|
||||
|
||||
def vars(self):
|
||||
return self.term.vars()
|
||||
|
||||
def map(self, map_):
|
||||
return self.lang[BooleanOp(self.term.map(map_))]
|
||||
|
||||
def missing(self):
|
||||
return self.term.missing()
|
||||
|
||||
@simplified
|
||||
def partial_eval(self):
|
||||
term = self.lang[self.term].partial_eval()
|
||||
if term is TRUE:
|
||||
return TRUE
|
||||
elif term in (FALSE, NULL):
|
||||
return FALSE
|
||||
elif term.type is BOOLEAN:
|
||||
return term
|
||||
elif term is self.term:
|
||||
return self
|
||||
|
||||
exists = self.lang[term].exists().partial_eval()
|
||||
return exists
|
||||
|
|
@ -0,0 +1,124 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions import first_op, not_op, eq_op
|
||||
from jx_base.expressions._utils import simplified
|
||||
from jx_base.expressions.and_op import AndOp
|
||||
from jx_base.expressions.not_op import NotOp
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.false_op import FALSE
|
||||
from jx_base.expressions.literal import NULL
|
||||
from jx_base.expressions.or_op import OrOp
|
||||
from jx_base.expressions.true_op import TRUE
|
||||
from jx_base.expressions.when_op import WhenOp
|
||||
from jx_base.language import is_op
|
||||
from mo_dots import is_sequence
|
||||
from mo_future import first
|
||||
from mo_json import OBJECT, BOOLEAN
|
||||
from mo_logs import Log
|
||||
|
||||
|
||||
class CaseOp(Expression):
|
||||
def __init__(self, terms, **clauses):
|
||||
if not is_sequence(terms):
|
||||
Log.error("case expression requires a list of `when` sub-clauses")
|
||||
Expression.__init__(self, terms)
|
||||
if len(terms) == 0:
|
||||
Log.error("Expecting at least one clause")
|
||||
|
||||
for w in terms[:-1]:
|
||||
if not is_op(w, WhenOp) or w.els_ is not NULL:
|
||||
Log.error(
|
||||
"case expression does not allow `else` clause in `when` sub-clause"
|
||||
)
|
||||
self.whens = terms
|
||||
|
||||
def __data__(self):
|
||||
return {"case": [w.__data__() for w in self.whens]}
|
||||
|
||||
def __eq__(self, other):
|
||||
if is_op(other, CaseOp):
|
||||
return all(s == o for s, o in zip(self.whens, other.whens))
|
||||
|
||||
def vars(self):
|
||||
output = set()
|
||||
for w in self.whens:
|
||||
output |= w.vars()
|
||||
return output
|
||||
|
||||
def map(self, map_):
|
||||
return self.lang[CaseOp([w.map(map_) for w in self.whens])]
|
||||
|
||||
def missing(self):
|
||||
m = self.whens[-1].missing()
|
||||
for w in reversed(self.whens[0:-1]):
|
||||
when = w.when.partial_eval()
|
||||
if when is FALSE:
|
||||
pass
|
||||
elif when is TRUE:
|
||||
m = w.then.partial_eval().missing()
|
||||
else:
|
||||
m = self.lang[OrOp([AndOp([when, w.then.partial_eval().missing()]), m])]
|
||||
return m.partial_eval()
|
||||
|
||||
@simplified
|
||||
def partial_eval(self):
|
||||
if self.type == BOOLEAN:
|
||||
nots = []
|
||||
ors = []
|
||||
for w in self.whens[:-1]:
|
||||
ors.append(AndOp(nots + [w.when, w.then]))
|
||||
nots.append(NotOp(w.when))
|
||||
ors.append(AndOp(nots + [self.whens[-1]]))
|
||||
return self.lang[OrOp(ors)].partial_eval()
|
||||
|
||||
whens = []
|
||||
for w in self.whens[:-1]:
|
||||
when = self.lang[w.when].partial_eval()
|
||||
if when is TRUE:
|
||||
whens.append(self.lang[w.then].partial_eval())
|
||||
break
|
||||
elif when is FALSE:
|
||||
pass
|
||||
else:
|
||||
whens.append(self.lang[WhenOp(when, **{"then": w.then.partial_eval()})])
|
||||
else:
|
||||
whens.append(self.lang[self.whens[-1]].partial_eval())
|
||||
|
||||
if len(whens) == 1:
|
||||
return whens[0]
|
||||
elif len(whens) == 2:
|
||||
return self.lang[WhenOp(whens[0].when, **{"then": whens[0].then, "else": whens[1]})]
|
||||
else:
|
||||
return self.lang[CaseOp(whens)]
|
||||
|
||||
@property
|
||||
def type(self):
|
||||
types = set(w.then.type if is_op(w, WhenOp) else w.type for w in self.whens)
|
||||
if len(types) > 1:
|
||||
return OBJECT
|
||||
else:
|
||||
return first(types)
|
||||
|
||||
|
||||
first_op.CaseOp = CaseOp
|
||||
not_op.CaseOp = CaseOp
|
||||
eq_op.CaseOp = CaseOp
|
||||
|
|
@ -0,0 +1,78 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions.literal import is_literal
|
||||
from jx_base.expressions.null_op import NULL
|
||||
from jx_base.expressions._utils import simplified
|
||||
from jx_base.expressions.and_op import AndOp
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.first_op import FirstOp
|
||||
from jx_base.language import is_op
|
||||
|
||||
|
||||
class CoalesceOp(Expression):
|
||||
has_simple_form = True
|
||||
|
||||
def __init__(self, terms):
|
||||
Expression.__init__(self, terms)
|
||||
self.terms = terms
|
||||
|
||||
def __data__(self):
|
||||
return {"coalesce": [t.__data__() for t in self.terms]}
|
||||
|
||||
def __eq__(self, other):
|
||||
if is_op(other, CoalesceOp):
|
||||
if len(self.terms) == len(other.terms):
|
||||
return all(s == o for s, o in zip(self.terms, other.terms))
|
||||
return False
|
||||
|
||||
def missing(self):
|
||||
# RETURN true FOR RECORDS THE WOULD RETURN NULL
|
||||
return self.lang[AndOp([v.missing() for v in self.terms])]
|
||||
|
||||
def vars(self):
|
||||
output = set()
|
||||
for v in self.terms:
|
||||
output |= v.vars()
|
||||
return output
|
||||
|
||||
def map(self, map_):
|
||||
return self.lang[CoalesceOp([v.map(map_) for v in self.terms])]
|
||||
|
||||
@simplified
|
||||
def partial_eval(self):
|
||||
terms = []
|
||||
for t in self.terms:
|
||||
simple = self.lang[FirstOp(t)].partial_eval()
|
||||
if simple is NULL:
|
||||
pass
|
||||
elif is_literal(simple):
|
||||
terms.append(simple)
|
||||
break
|
||||
else:
|
||||
terms.append(simple)
|
||||
|
||||
if len(terms) == 0:
|
||||
return NULL
|
||||
elif len(terms) == 1:
|
||||
return terms[0]
|
||||
else:
|
||||
return self.lang[CoalesceOp(terms)]
|
||||
|
|
@ -0,0 +1,86 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions._utils import jx_expression
|
||||
from jx_base.expressions.and_op import AndOp
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.literal import Literal
|
||||
from jx_base.expressions.literal import is_literal
|
||||
from jx_base.expressions.null_op import NULL
|
||||
from jx_base.expressions.variable import Variable
|
||||
from jx_base.language import is_op
|
||||
from jx_base.utils import is_variable_name
|
||||
from mo_dots import is_data
|
||||
from mo_future import first, is_text
|
||||
from mo_json import STRING
|
||||
from mo_logs import Log
|
||||
|
||||
|
||||
class ConcatOp(Expression):
|
||||
has_simple_form = True
|
||||
data_type = STRING
|
||||
|
||||
def __init__(self, terms, **clauses):
|
||||
Expression.__init__(self, terms)
|
||||
if is_data(terms):
|
||||
self.terms = first(terms.items())
|
||||
else:
|
||||
self.terms = terms
|
||||
self.separator = clauses.get(str("separator"), Literal(""))
|
||||
self.default = clauses.get(str("default"), NULL)
|
||||
if not is_literal(self.separator):
|
||||
Log.error("Expecting a literal separator")
|
||||
|
||||
@classmethod
|
||||
def define(cls, expr):
|
||||
terms = expr["concat"]
|
||||
if is_data(terms):
|
||||
k, v = first(terms.items())
|
||||
terms = [Variable(k), Literal(v)]
|
||||
else:
|
||||
terms = [jx_expression(t) for t in terms]
|
||||
|
||||
return cls.lang[
|
||||
ConcatOp(
|
||||
terms,
|
||||
**{
|
||||
k: Literal(v)
|
||||
if is_text(v) and not is_variable_name(v)
|
||||
else jx_expression(v)
|
||||
for k, v in expr.items()
|
||||
if k in ["default", "separator"]
|
||||
}
|
||||
)
|
||||
]
|
||||
|
||||
def __data__(self):
|
||||
f, s = self.terms[0], self.terms[1]
|
||||
if is_op(f, Variable) and is_literal(s):
|
||||
output = {"concat": {f.var: s.value}}
|
||||
else:
|
||||
output = {"concat": [t.__data__() for t in self.terms]}
|
||||
if self.separator.json != '""':
|
||||
output["separator"] = self.separator.__data__()
|
||||
return output
|
||||
|
||||
def vars(self):
|
||||
if not self.terms:
|
||||
return set()
|
||||
return set.union(*(t.vars() for t in self.terms))
|
||||
|
||||
def map(self, map_):
|
||||
return self.lang[
|
||||
ConcatOp([t.map(map_) for t in self.terms], separator=self.separator)
|
||||
]
|
||||
|
||||
def missing(self):
|
||||
return self.lang[
|
||||
AndOp([t.missing() for t in self.terms] + [self.default.missing()])
|
||||
].partial_eval()
|
||||
|
|
@ -0,0 +1,55 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions.false_op import FALSE
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.true_op import TrueOp
|
||||
from jx_base.expressions.tuple_op import TupleOp
|
||||
from mo_dots import is_many
|
||||
from mo_json import INTEGER
|
||||
|
||||
|
||||
class CountOp(Expression):
|
||||
has_simple_form = False
|
||||
data_type = INTEGER
|
||||
|
||||
def __init__(self, terms, **clauses):
|
||||
Expression.__init__(self, terms)
|
||||
if is_many(terms):
|
||||
# SHORTCUT: ASSUME AN ARRAY OF IS A TUPLE
|
||||
self.terms = self.lang[TupleOp(terms)]
|
||||
else:
|
||||
self.terms = terms
|
||||
|
||||
def __data__(self):
|
||||
return {"count": self.terms.__data__()}
|
||||
|
||||
def vars(self):
|
||||
return self.terms.vars()
|
||||
|
||||
def map(self, map_):
|
||||
return self.lang[CountOp(self.terms.map(map_))]
|
||||
|
||||
def missing(self):
|
||||
return FALSE
|
||||
|
||||
def exists(self):
|
||||
return TrueOp
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions import literal
|
||||
from jx_base.expressions.literal import Literal
|
||||
from mo_dots import coalesce
|
||||
from mo_future import is_text
|
||||
from mo_json import NUMBER
|
||||
from mo_times.dates import unicode2Date, Date
|
||||
|
||||
|
||||
class DateOp(Literal):
|
||||
date_type = NUMBER
|
||||
|
||||
def __init__(self, term):
|
||||
if hasattr(self, "date"):
|
||||
return
|
||||
if is_text(term):
|
||||
self.date = term
|
||||
else:
|
||||
self.date = coalesce(term.get("literal"), term)
|
||||
v = unicode2Date(self.date)
|
||||
if isinstance(v, Date):
|
||||
Literal.__init__(self, v.unix)
|
||||
else:
|
||||
Literal.__init__(self, v.seconds)
|
||||
|
||||
@classmethod
|
||||
def define(cls, expr):
|
||||
return cls.lang[DateOp(expr.get("date"))]
|
||||
|
||||
def __data__(self):
|
||||
return {"date": self.date}
|
||||
|
||||
def __call__(self, row=None, rownum=None, rows=None):
|
||||
return Date(self.date)
|
||||
|
||||
|
||||
literal.DateOp=DateOp
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions._utils import simplified, builtin_ops
|
||||
from jx_base.expressions.and_op import AndOp
|
||||
from jx_base.expressions.base_binary_op import BaseBinaryOp
|
||||
from jx_base.expressions.eq_op import EqOp
|
||||
from jx_base.expressions.literal import Literal, ZERO, is_literal
|
||||
from jx_base.expressions.or_op import OrOp
|
||||
|
||||
|
||||
class DivOp(BaseBinaryOp):
|
||||
op = "div"
|
||||
|
||||
def missing(self):
|
||||
return self.lang[
|
||||
AndOp(
|
||||
[
|
||||
self.default.missing(),
|
||||
OrOp(
|
||||
[self.lhs.missing(), self.rhs.missing(), EqOp([self.rhs, ZERO])]
|
||||
),
|
||||
]
|
||||
)
|
||||
].partial_eval()
|
||||
|
||||
@simplified
|
||||
def partial_eval(self):
|
||||
default = self.default.partial_eval()
|
||||
rhs = self.rhs.partial_eval()
|
||||
if rhs is ZERO:
|
||||
return default
|
||||
lhs = self.lhs.partial_eval()
|
||||
if is_literal(lhs) and is_literal(rhs):
|
||||
return Literal(builtin_ops[self.op](lhs.value, rhs.value))
|
||||
return self.__class__([lhs, rhs], default=default)
|
||||
|
|
@ -0,0 +1,104 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions._utils import simplified
|
||||
from jx_base.expressions.and_op import AndOp
|
||||
from jx_base.expressions.basic_eq_op import BasicEqOp
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.false_op import FALSE
|
||||
from jx_base.expressions.literal import is_literal
|
||||
from jx_base.expressions.true_op import TRUE
|
||||
from jx_base.expressions.variable import Variable
|
||||
from jx_base.language import is_op, value_compare
|
||||
from mo_dots import is_many
|
||||
from mo_json import BOOLEAN
|
||||
|
||||
CaseOp = None
|
||||
InOp = None
|
||||
WhneOp = None
|
||||
|
||||
class EqOp(Expression):
|
||||
has_simple_form = True
|
||||
data_type = BOOLEAN
|
||||
|
||||
def __new__(cls, terms):
|
||||
if is_many(terms):
|
||||
return object.__new__(cls)
|
||||
|
||||
items = terms.items()
|
||||
if len(items) == 1:
|
||||
if is_many(items[0][1]):
|
||||
return cls.lang[InOp(items[0])]
|
||||
else:
|
||||
return cls.lang[EqOp(items[0])]
|
||||
else:
|
||||
acc = []
|
||||
for lhs, rhs in items:
|
||||
if rhs.json.startswith("["):
|
||||
acc.append(cls.lang[InOp([Variable(lhs), rhs])])
|
||||
else:
|
||||
acc.append(cls.lang[EqOp([Variable(lhs), rhs])])
|
||||
return cls.lang[AndOp(acc)]
|
||||
|
||||
def __init__(self, terms):
|
||||
Expression.__init__(self, terms)
|
||||
self.lhs, self.rhs = terms
|
||||
|
||||
def __data__(self):
|
||||
if is_op(self.lhs, Variable) and is_literal(self.rhs):
|
||||
return {"eq": {self.lhs.var, self.rhs.value}}
|
||||
else:
|
||||
return {"eq": [self.lhs.__data__(), self.rhs.__data__()]}
|
||||
|
||||
def __eq__(self, other):
|
||||
if is_op(other, EqOp):
|
||||
return self.lhs == other.lhs and self.rhs == other.rhs
|
||||
return False
|
||||
|
||||
def vars(self):
|
||||
return self.lhs.vars() | self.rhs.vars()
|
||||
|
||||
def map(self, map_):
|
||||
return self.lang[EqOp([self.lhs.map(map_), self.rhs.map(map_)])]
|
||||
|
||||
def missing(self):
|
||||
return FALSE
|
||||
|
||||
def exists(self):
|
||||
return TRUE
|
||||
|
||||
@simplified
|
||||
def partial_eval(self):
|
||||
lhs = self.lang[self.lhs].partial_eval()
|
||||
rhs = self.lang[self.rhs].partial_eval()
|
||||
|
||||
if is_literal(lhs) and is_literal(rhs):
|
||||
return FALSE if value_compare(lhs.value, rhs.value) else TRUE
|
||||
else:
|
||||
return self.lang[
|
||||
self.lang[CaseOp(
|
||||
[
|
||||
WhenOp(lhs.missing(), **{"then": rhs.missing()}),
|
||||
WhenOp(rhs.missing(), **{"then": FALSE}),
|
||||
BasicEqOp([lhs, rhs]),
|
||||
]
|
||||
)]
|
||||
].partial_eval()
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions._utils import simplified
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.language import is_op
|
||||
from mo_json import BOOLEAN
|
||||
|
||||
|
||||
class EsNestedOp(Expression):
|
||||
data_type = BOOLEAN
|
||||
has_simple_form = False
|
||||
|
||||
def __init__(self, terms):
|
||||
Expression.__init__(self, terms)
|
||||
self.path, self.query = terms
|
||||
|
||||
@simplified
|
||||
def partial_eval(self):
|
||||
if self.path.var == ".":
|
||||
return self.query.partial_eval()
|
||||
return self.lang[
|
||||
EsNestedOp("es.nested", [self.path, self.query.partial_eval()])
|
||||
]
|
||||
|
||||
def __data__(self):
|
||||
return {"es.nested": {self.path.var: self.query.__data__()}}
|
||||
|
||||
def __eq__(self, other):
|
||||
if is_op(other, EsNestedOp):
|
||||
return self.path.var == other.path.var and self.query == other.query
|
||||
return False
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions.expression import Expression
|
||||
|
||||
|
||||
class EsScript(Expression):
|
||||
"""
|
||||
REPRESENT A Painless SCRIPT
|
||||
"""
|
||||
|
||||
pass
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions._utils import simplified
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.false_op import FALSE
|
||||
from mo_json import BOOLEAN
|
||||
|
||||
NotOp = None
|
||||
|
||||
|
||||
class ExistsOp(Expression):
|
||||
data_type = BOOLEAN
|
||||
|
||||
def __init__(self, term):
|
||||
Expression.__init__(self, [term])
|
||||
self.field = term
|
||||
|
||||
def __data__(self):
|
||||
return {"exists": self.field.__data__()}
|
||||
|
||||
def vars(self):
|
||||
return self.field.vars()
|
||||
|
||||
def map(self, map_):
|
||||
return self.lang[ExistsOp(self.field.map(map_))]
|
||||
|
||||
def missing(self):
|
||||
return FALSE
|
||||
|
||||
def exists(self):
|
||||
return TRUE
|
||||
|
||||
@simplified
|
||||
def partial_eval(self):
|
||||
return self.lang[NotOp(self.field.missing())].partial_eval()
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions.base_binary_op import BaseBinaryOp
|
||||
|
||||
|
||||
class ExpOp(BaseBinaryOp):
|
||||
op = "exp"
|
||||
|
|
@ -0,0 +1,178 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions._utils import operators, jx_expression, _jx_expression, simplified
|
||||
from jx_base.language import BaseExpression, ID, is_expression, is_op
|
||||
from mo_dots import is_data, is_sequence, is_container
|
||||
from mo_future import items as items_, text
|
||||
from mo_json import BOOLEAN, OBJECT, value2json
|
||||
from mo_logs import Log
|
||||
|
||||
FALSE, Literal, is_literal, MissingOp, NotOp, NULL, Variable = [None]*7
|
||||
|
||||
|
||||
class Expression(BaseExpression):
|
||||
data_type = OBJECT
|
||||
has_simple_form = False
|
||||
|
||||
def __init__(self, args):
|
||||
self.simplified = False
|
||||
# SOME BASIC VERIFICATION THAT THESE ARE REASONABLE PARAMETERS
|
||||
if is_sequence(args):
|
||||
bad = [t for t in args if t != None and not is_expression(t)]
|
||||
if bad:
|
||||
Log.error("Expecting an expression, not {{bad}}", bad=bad)
|
||||
elif is_data(args):
|
||||
if not all(is_op(k, Variable) and is_literal(v) for k, v in args.items()):
|
||||
Log.error("Expecting an {<variable>: <literal>}")
|
||||
elif args == None:
|
||||
pass
|
||||
else:
|
||||
if not is_expression(args):
|
||||
Log.error("Expecting an expression")
|
||||
|
||||
@classmethod
|
||||
def get_id(cls):
|
||||
return getattr(cls, ID)
|
||||
|
||||
@classmethod
|
||||
def define(cls, expr):
|
||||
"""
|
||||
GENERAL SUPPORT FOR BUILDING EXPRESSIONS FROM JSON EXPRESSIONS
|
||||
OVERRIDE THIS IF AN OPERATOR EXPECTS COMPLICATED PARAMETERS
|
||||
:param expr: Data representing a JSON Expression
|
||||
:return: parse tree
|
||||
"""
|
||||
|
||||
try:
|
||||
lang = cls.lang
|
||||
items = items_(expr)
|
||||
for item in items:
|
||||
op, term = item
|
||||
full_op = operators.get(op)
|
||||
if full_op:
|
||||
class_ = lang.ops[full_op.get_id()]
|
||||
clauses = {k: jx_expression(v) for k, v in expr.items() if k != op}
|
||||
break
|
||||
else:
|
||||
if not items:
|
||||
return NULL
|
||||
raise Log.error(
|
||||
"{{operator|quote}} is not a known operator", operator=expr
|
||||
)
|
||||
|
||||
if term == None:
|
||||
return class_([], **clauses)
|
||||
elif is_container(term):
|
||||
terms = [jx_expression(t) for t in term]
|
||||
return class_(terms, **clauses)
|
||||
elif is_data(term):
|
||||
items = items_(term)
|
||||
if class_.has_simple_form:
|
||||
if len(items) == 1:
|
||||
k, v = items[0]
|
||||
return class_([Variable(k), Literal(v)], **clauses)
|
||||
else:
|
||||
return class_({k: Literal(v) for k, v in items}, **clauses)
|
||||
else:
|
||||
return class_(_jx_expression(term, lang), **clauses)
|
||||
else:
|
||||
if op in ["literal", "date", "offset"]:
|
||||
return class_(term, **clauses)
|
||||
else:
|
||||
return class_(_jx_expression(term, lang), **clauses)
|
||||
except Exception as e:
|
||||
Log.error("programmer error expr = {{value|quote}}", value=expr, cause=e)
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
return self.__class__.__name__
|
||||
|
||||
@property
|
||||
def many(self):
|
||||
"""
|
||||
:return: True IF THE EXPRESSION RETURNS A MULTIVALUE (WHICH IS NOT A LIST OR A TUPLE)
|
||||
"""
|
||||
return False
|
||||
|
||||
def __data__(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def vars(self):
|
||||
raise Log.error("{{type}} has no `vars` method", type=self.__class__.__name__)
|
||||
|
||||
def map(self, map):
|
||||
raise Log.error("{{type}} has no `map` method", type=self.__class__.__name__)
|
||||
|
||||
def missing(self):
|
||||
"""
|
||||
THERE IS PLENTY OF OPPORTUNITY TO SIMPLIFY missing EXPRESSIONS
|
||||
OVERRIDE THIS METHOD TO SIMPLIFY
|
||||
:return:
|
||||
"""
|
||||
if self.type == BOOLEAN:
|
||||
Log.error("programmer error")
|
||||
return self.lang[MissingOp(self)]
|
||||
|
||||
def exists(self):
|
||||
"""
|
||||
THERE IS PLENTY OF OPPORTUNITY TO SIMPLIFY exists EXPRESSIONS
|
||||
OVERRIDE THIS METHOD TO SIMPLIFY
|
||||
:return:
|
||||
"""
|
||||
return self.lang[NotOp(self.missing()).partial_eval()]
|
||||
|
||||
def is_true(self):
|
||||
"""
|
||||
:return: True, IF THIS EXPRESSION ALWAYS RETURNS BOOLEAN true
|
||||
"""
|
||||
return FALSE # GOOD DEFAULT ASSUMPTION
|
||||
|
||||
def is_false(self):
|
||||
"""
|
||||
:return: True, IF THIS EXPRESSION ALWAYS RETURNS BOOLEAN false
|
||||
"""
|
||||
return FALSE # GOOD DEFAULT ASSUMPTION
|
||||
|
||||
@simplified
|
||||
def partial_eval(self):
|
||||
"""
|
||||
ATTEMPT TO SIMPLIFY THE EXPRESSION:
|
||||
PREFERABLY RETURNING A LITERAL, BUT MAYBE A SIMPLER EXPRESSION, OR self IF NOT POSSIBLE
|
||||
"""
|
||||
return self
|
||||
|
||||
@property
|
||||
def type(self):
|
||||
return self.data_type
|
||||
|
||||
def __eq__(self, other):
|
||||
if other is None:
|
||||
return False
|
||||
if self.get_id() != other.get_id():
|
||||
return False
|
||||
self_class = self.__class__
|
||||
Log.note("this is slow on {{type}}", type=text(self_class.__name__))
|
||||
return self.__data__() == other.__data__()
|
||||
|
||||
def __str__(self):
|
||||
return value2json(self.__data__(), pretty=True)
|
||||
|
||||
|
|
@ -0,0 +1,82 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions import literal, expression
|
||||
from jx_base.expressions.literal import Literal
|
||||
from mo_json import BOOLEAN
|
||||
|
||||
TRUE = None
|
||||
|
||||
|
||||
class FalseOp(Literal):
|
||||
data_type = BOOLEAN
|
||||
|
||||
def __new__(cls, *args, **kwargs):
|
||||
return object.__new__(cls, *args, **kwargs)
|
||||
|
||||
def __init__(self, op=None, term=None):
|
||||
Literal.__init__(self, False)
|
||||
|
||||
@classmethod
|
||||
def define(cls, expr):
|
||||
return FALSE
|
||||
|
||||
def __nonzero__(self):
|
||||
return False
|
||||
|
||||
def __eq__(self, other):
|
||||
return (other is FALSE) or (other is False)
|
||||
|
||||
def __data__(self):
|
||||
return False
|
||||
|
||||
def vars(self):
|
||||
return set()
|
||||
|
||||
def map(self, map_):
|
||||
return self
|
||||
|
||||
def missing(self):
|
||||
return FALSE
|
||||
|
||||
def is_true(self):
|
||||
return FALSE
|
||||
|
||||
def is_false(self):
|
||||
return TRUE
|
||||
|
||||
def __call__(self, row=None, rownum=None, rows=None):
|
||||
return False
|
||||
|
||||
def __unicode__(self):
|
||||
return "false"
|
||||
|
||||
def __str__(self):
|
||||
return b"false"
|
||||
|
||||
def __bool__(self):
|
||||
return False
|
||||
|
||||
|
||||
FALSE = FalseOp()
|
||||
|
||||
expression.FALSE = FALSE
|
||||
literal.FALSE = FALSE
|
||||
|
|
@ -0,0 +1,75 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.literal import ZERO
|
||||
from jx_base.expressions.literal import is_literal
|
||||
from jx_base.expressions.null_op import NULL
|
||||
from jx_base.expressions.variable import Variable
|
||||
from jx_base.language import is_op
|
||||
from mo_json import INTEGER
|
||||
|
||||
|
||||
class FindOp(Expression):
|
||||
"""
|
||||
RETURN INDEX OF find IN value, ELSE RETURN null
|
||||
"""
|
||||
|
||||
has_simple_form = True
|
||||
data_type = INTEGER
|
||||
|
||||
def __init__(self, term, **kwargs):
|
||||
Expression.__init__(self, term)
|
||||
self.value, self.find = term
|
||||
self.default = kwargs.get("default", NULL)
|
||||
self.start = kwargs.get("start", ZERO).partial_eval()
|
||||
if self.start is NULL:
|
||||
self.start = ZERO
|
||||
|
||||
def __data__(self):
|
||||
if is_op(self.value, Variable) and is_literal(self.find):
|
||||
output = {
|
||||
"find": {self.value.var, self.find.value},
|
||||
"start": self.start.__data__(),
|
||||
}
|
||||
else:
|
||||
output = {
|
||||
"find": [self.value.__data__(), self.find.__data__()],
|
||||
"start": self.start.__data__(),
|
||||
}
|
||||
if self.default is not NULL:
|
||||
output["default"] = self.default.__data__()
|
||||
return output
|
||||
|
||||
def vars(self):
|
||||
return (
|
||||
self.value.vars()
|
||||
| self.find.vars()
|
||||
| self.default.vars()
|
||||
| self.start.vars()
|
||||
)
|
||||
|
||||
def map(self, map_):
|
||||
return FindOp(
|
||||
[self.value.map(map_), self.find.map(map_)],
|
||||
start=self.start.map(map_),
|
||||
default=self.default.map(map_),
|
||||
)
|
||||
|
|
@ -0,0 +1,79 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions._utils import simplified
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.last_op import LastOp
|
||||
from jx_base.expressions.literal import is_literal
|
||||
from jx_base.language import is_op
|
||||
from mo_json import OBJECT
|
||||
from mo_logs import Log
|
||||
|
||||
CaseOp = None
|
||||
WhenOp = None
|
||||
|
||||
|
||||
class FirstOp(Expression):
|
||||
def __init__(self, term):
|
||||
Expression.__init__(self, [term])
|
||||
self.term = term
|
||||
self.data_type = self.term.type
|
||||
|
||||
def __data__(self):
|
||||
return {"first": self.term.__data__()}
|
||||
|
||||
def vars(self):
|
||||
return self.term.vars()
|
||||
|
||||
def map(self, map_):
|
||||
return self.lang[LastOp(self.term.map(map_))]
|
||||
|
||||
def missing(self):
|
||||
return self.term.missing()
|
||||
|
||||
@simplified
|
||||
def partial_eval(self):
|
||||
term = self.lang[self.term].partial_eval()
|
||||
if is_op(term, FirstOp):
|
||||
return term
|
||||
elif is_op(term, CaseOp): # REWRITING
|
||||
return self.lang[
|
||||
CaseOp(
|
||||
[
|
||||
WhenOp(t.when, **{"then": FirstOp(t.then)})
|
||||
for t in term.whens[:-1]
|
||||
]
|
||||
+ [FirstOp(term.whens[-1])]
|
||||
)
|
||||
].partial_eval()
|
||||
elif is_op(term, WhenOp):
|
||||
return self.lang[
|
||||
WhenOp(
|
||||
term.when,
|
||||
**{"then": FirstOp(term.then), "else": FirstOp(term.els_)}
|
||||
)
|
||||
].partial_eval()
|
||||
elif term.type != OBJECT and not term.many:
|
||||
return term
|
||||
elif is_literal(term):
|
||||
Log.error("not handled yet")
|
||||
else:
|
||||
return self.lang[FirstOp(term)]
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions.eq_op import EqOp
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.false_op import FALSE
|
||||
from jx_base.expressions.literal import ZERO, ONE
|
||||
from jx_base.expressions.literal import is_literal
|
||||
from jx_base.expressions.null_op import NULL
|
||||
from jx_base.expressions.or_op import OrOp
|
||||
from jx_base.expressions.variable import Variable
|
||||
from jx_base.language import is_op
|
||||
from mo_json import NUMBER
|
||||
|
||||
|
||||
class FloorOp(Expression):
|
||||
has_simple_form = True
|
||||
data_type = NUMBER
|
||||
|
||||
def __init__(self, terms, default=NULL):
|
||||
Expression.__init__(self, terms)
|
||||
if len(terms) == 1:
|
||||
self.lhs = terms[0]
|
||||
self.rhs = ONE
|
||||
else:
|
||||
self.lhs, self.rhs = terms
|
||||
self.default = default
|
||||
|
||||
def __data__(self):
|
||||
if is_op(self.lhs, Variable) and is_literal(self.rhs):
|
||||
return {"floor": {self.lhs.var, self.rhs.value}, "default": self.default}
|
||||
else:
|
||||
return {
|
||||
"floor": [self.lhs.__data__(), self.rhs.__data__()],
|
||||
"default": self.default,
|
||||
}
|
||||
|
||||
def vars(self):
|
||||
return self.lhs.vars() | self.rhs.vars() | self.default.vars()
|
||||
|
||||
def map(self, map_):
|
||||
return self.lang[
|
||||
FloorOp(
|
||||
[self.lhs.map(map_), self.rhs.map(map_)], default=self.default.map(map_)
|
||||
)
|
||||
]
|
||||
|
||||
def missing(self):
|
||||
if self.default.exists():
|
||||
return FALSE
|
||||
else:
|
||||
return self.lang[
|
||||
OrOp([self.lhs.missing(), self.rhs.missing(), EqOp([self.rhs, ZERO])])
|
||||
]
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions.expression import Expression
|
||||
from mo_json import NUMBER
|
||||
|
||||
|
||||
class FromUnixOp(Expression):
|
||||
"""
|
||||
FOR USING ON DATABASES WHICH HAVE A DATE COLUMNS: CONVERT TO UNIX
|
||||
"""
|
||||
|
||||
data_type = NUMBER
|
||||
|
||||
def __init__(self, term):
|
||||
Expression.__init__(self, term)
|
||||
self.value = term
|
||||
|
||||
def vars(self):
|
||||
return self.value.vars()
|
||||
|
||||
def map(self, map_):
|
||||
return self.lang[FromUnixOp(self.value.map(map_))]
|
||||
|
||||
def missing(self):
|
||||
return self.value.missing()
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.literal import is_literal
|
||||
|
||||
|
||||
class GetOp(Expression):
|
||||
has_simple_form = True
|
||||
|
||||
def __init__(self, term):
|
||||
Expression.__init__(self, term)
|
||||
self.var = term[0]
|
||||
self.offsets = term[1:]
|
||||
|
||||
def __data__(self):
|
||||
if is_literal(self.var) and len(self.offsets) == 1 and is_literal(self.offset):
|
||||
return {"get": {self.var.json, self.offsets[0].value}}
|
||||
else:
|
||||
return {"get": [self.var.__data__()] + [o.__data__() for o in self.offsets]}
|
||||
|
||||
def vars(self):
|
||||
output = self.var.vars()
|
||||
for o in self.offsets:
|
||||
output |= o.vars()
|
||||
return output
|
||||
|
||||
def map(self, map_):
|
||||
return self.lang[
|
||||
GetOp([self.var.map(map_)] + [o.map(map_) for o in self.offsets])
|
||||
]
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions.base_inequality_op import BaseInequalityOp
|
||||
|
||||
|
||||
class GtOp(BaseInequalityOp):
|
||||
op = "gt"
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions.base_inequality_op import BaseInequalityOp
|
||||
|
||||
|
||||
class GteOp(BaseInequalityOp):
|
||||
op = "gte"
|
||||
|
|
@ -0,0 +1,85 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions import eq_op
|
||||
from jx_base.expressions._utils import simplified
|
||||
from jx_base.expressions.eq_op import EqOp
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.false_op import FALSE
|
||||
from jx_base.expressions.literal import Literal
|
||||
from jx_base.expressions.literal import is_literal
|
||||
from jx_base.expressions.null_op import NULL
|
||||
from jx_base.expressions.variable import Variable
|
||||
from jx_base.language import is_op
|
||||
from mo_dots import is_many
|
||||
from mo_json import BOOLEAN
|
||||
|
||||
|
||||
class InOp(Expression):
|
||||
has_simple_form = True
|
||||
data_type = BOOLEAN
|
||||
|
||||
def __new__(cls, terms):
|
||||
if is_op(terms[0], Variable) and is_op(terms[1], Literal):
|
||||
name, value = terms
|
||||
if not is_many(value.value):
|
||||
return cls.lang[EqOp([name, Literal([value.value])])]
|
||||
return object.__new__(cls)
|
||||
|
||||
def __init__(self, term):
|
||||
Expression.__init__(self, term)
|
||||
self.value, self.superset = term
|
||||
|
||||
def __data__(self):
|
||||
if is_op(self.value, Variable) and is_literal(self.superset):
|
||||
return {"in": {self.value.var: self.superset.value}}
|
||||
else:
|
||||
return {"in": [self.value.__data__(), self.superset.__data__()]}
|
||||
|
||||
def __eq__(self, other):
|
||||
if is_op(other, InOp):
|
||||
return self.value == other.value and self.superset == other.superset
|
||||
return False
|
||||
|
||||
def vars(self):
|
||||
return self.value.vars()
|
||||
|
||||
def map(self, map_):
|
||||
return self.lang[InOp([self.value.map(map_), self.superset.map(map_)])]
|
||||
|
||||
@simplified
|
||||
def partial_eval(self):
|
||||
value = self.value.partial_eval()
|
||||
superset = self.superset.partial_eval()
|
||||
if superset is NULL:
|
||||
return FALSE
|
||||
elif is_literal(value) and is_literal(superset):
|
||||
return self.lang[Literal(self())]
|
||||
else:
|
||||
return self.lang[InOp([value, superset])]
|
||||
|
||||
def __call__(self):
|
||||
return self.value() in self.superset()
|
||||
|
||||
def missing(self):
|
||||
return FALSE
|
||||
|
||||
eq_op.InOp = InOp
|
||||
|
|
@ -0,0 +1,56 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions._utils import simplified
|
||||
from jx_base.expressions.coalesce_op import CoalesceOp
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.first_op import FirstOp
|
||||
from jx_base.language import is_op
|
||||
from mo_json import INTEGER
|
||||
|
||||
|
||||
class IntegerOp(Expression):
|
||||
data_type = INTEGER
|
||||
|
||||
def __init__(self, term):
|
||||
Expression.__init__(self, [term])
|
||||
self.term = term
|
||||
|
||||
def __data__(self):
|
||||
return {"integer": self.term.__data__()}
|
||||
|
||||
def vars(self):
|
||||
return self.term.vars()
|
||||
|
||||
def map(self, map_):
|
||||
return self.lang[IntegerOp(self.term.map(map_))]
|
||||
|
||||
def missing(self):
|
||||
return self.term.missing()
|
||||
|
||||
@simplified
|
||||
def partial_eval(self):
|
||||
term = self.lang[FirstOp(self.term)].partial_eval()
|
||||
if is_op(term, CoalesceOp):
|
||||
return self.lang[CoalesceOp([IntegerOp(t) for t in term.terms])]
|
||||
if term.type == INTEGER:
|
||||
return term
|
||||
return self.lang[IntegerOp(term)]
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.false_op import FALSE
|
||||
from mo_json import BOOLEAN
|
||||
|
||||
|
||||
class IsBooleanOp(Expression):
|
||||
data_type = BOOLEAN
|
||||
|
||||
def __init__(self, term):
|
||||
Expression.__init__(self, [term])
|
||||
self.term = term
|
||||
|
||||
def __data__(self):
|
||||
return {"is_boolean": self.term.__data__()}
|
||||
|
||||
def vars(self):
|
||||
return self.term.vars()
|
||||
|
||||
def map(self, map_):
|
||||
return self.lang[IsBooleanOp(self.term.map(map_))]
|
||||
|
||||
def missing(self):
|
||||
return FALSE
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.false_op import FALSE
|
||||
from mo_json import BOOLEAN
|
||||
|
||||
|
||||
class IsIntegerOp(Expression):
|
||||
data_type = BOOLEAN
|
||||
|
||||
def __init__(self, term):
|
||||
Expression.__init__(self, [term])
|
||||
self.term = term
|
||||
|
||||
def __data__(self):
|
||||
return {"is_integer": self.term.__data__()}
|
||||
|
||||
def vars(self):
|
||||
return self.term.vars()
|
||||
|
||||
def map(self, map_):
|
||||
return self.lang[IsIntegerOp(self.term.map(map_))]
|
||||
|
||||
def missing(self):
|
||||
return FALSE
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions._utils import simplified
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.false_op import FALSE
|
||||
from jx_base.expressions.null_op import NULL
|
||||
from jx_base.expressions.true_op import TRUE
|
||||
from mo_json import BOOLEAN, INTEGER, NUMBER, OBJECT, NUMBER_TYPES
|
||||
|
||||
|
||||
class IsNumberOp(Expression):
|
||||
data_type = BOOLEAN
|
||||
|
||||
def __init__(self, term):
|
||||
Expression.__init__(self, [term])
|
||||
self.term = term
|
||||
|
||||
def __data__(self):
|
||||
return {"is_number": self.term.__data__()}
|
||||
|
||||
def vars(self):
|
||||
return self.term.vars()
|
||||
|
||||
def map(self, map_):
|
||||
return self.lang[IsNumberOp(self.term.map(map_))]
|
||||
|
||||
def missing(self):
|
||||
return FALSE
|
||||
|
||||
@simplified
|
||||
def partial_eval(self):
|
||||
term = self.term.partial_eval()
|
||||
|
||||
if term is NULL:
|
||||
return FALSE
|
||||
elif term.type in NUMBER_TYPES:
|
||||
return TRUE
|
||||
elif term.type == OBJECT:
|
||||
return self
|
||||
else:
|
||||
return FALSE
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.false_op import FALSE
|
||||
from mo_json import BOOLEAN
|
||||
|
||||
|
||||
class IsStringOp(Expression):
|
||||
data_type = BOOLEAN
|
||||
|
||||
def __init__(self, term):
|
||||
Expression.__init__(self, [term])
|
||||
self.term = term
|
||||
|
||||
def __data__(self):
|
||||
return {"is_string": self.term.__data__()}
|
||||
|
||||
def vars(self):
|
||||
return self.term.vars()
|
||||
|
||||
def map(self, map_):
|
||||
return self.lang[IsStringOp(self.term.map(map_))]
|
||||
|
||||
def missing(self):
|
||||
return FALSE
|
||||
|
|
@ -0,0 +1,62 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions._utils import simplified
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.literal import is_literal
|
||||
from jx_base.expressions.null_op import NULL
|
||||
from jx_base.language import is_op
|
||||
from mo_dots import is_many
|
||||
from mo_dots.lists import last
|
||||
from mo_json import OBJECT
|
||||
|
||||
|
||||
class LastOp(Expression):
|
||||
def __init__(self, term):
|
||||
Expression.__init__(self, [term])
|
||||
self.term = term
|
||||
self.data_type = self.term.type
|
||||
|
||||
def __data__(self):
|
||||
return {"last": self.term.__data__()}
|
||||
|
||||
def vars(self):
|
||||
return self.term.vars()
|
||||
|
||||
def map(self, map_):
|
||||
return self.lang[LastOp(self.term.map(map_))]
|
||||
|
||||
def missing(self):
|
||||
return self.term.missing()
|
||||
|
||||
@simplified
|
||||
def partial_eval(self):
|
||||
term = self.term.partial_eval()
|
||||
if is_op(self.term, LastOp):
|
||||
return term
|
||||
elif term.type != OBJECT and not term.many:
|
||||
return term
|
||||
elif term is NULL:
|
||||
return term
|
||||
elif is_literal(term):
|
||||
return last(term)
|
||||
else:
|
||||
return self.lang[LastOp(term)]
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.false_op import FALSE
|
||||
from mo_json import OBJECT
|
||||
|
||||
|
||||
class LeavesOp(Expression):
|
||||
date_type = OBJECT
|
||||
|
||||
def __init__(self, term, prefix=None):
|
||||
Expression.__init__(self, term)
|
||||
self.term = term
|
||||
self.prefix = prefix
|
||||
|
||||
def __data__(self):
|
||||
if self.prefix:
|
||||
return {"leaves": self.term.__data__(), "prefix": self.prefix}
|
||||
else:
|
||||
return {"leaves": self.term.__data__()}
|
||||
|
||||
def vars(self):
|
||||
return self.term.vars()
|
||||
|
||||
def map(self, map_):
|
||||
return self.lang[LeavesOp(self.term.map(map_))]
|
||||
|
||||
def missing(self):
|
||||
return FALSE
|
||||
|
|
@ -0,0 +1,81 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions._utils import simplified
|
||||
from jx_base.expressions.basic_substring_op import BasicSubstringOp
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.length_op import LengthOp
|
||||
from jx_base.expressions.literal import ZERO
|
||||
from jx_base.expressions.literal import is_literal
|
||||
from jx_base.expressions.max_op import MaxOp
|
||||
from jx_base.expressions.min_op import MinOp
|
||||
from jx_base.expressions.or_op import OrOp
|
||||
from jx_base.expressions.variable import Variable
|
||||
from jx_base.expressions.when_op import WhenOp
|
||||
from jx_base.language import is_op
|
||||
from mo_dots import is_data
|
||||
from mo_json import STRING
|
||||
|
||||
|
||||
class LeftOp(Expression):
|
||||
has_simple_form = True
|
||||
data_type = STRING
|
||||
|
||||
def __init__(self, term):
|
||||
Expression.__init__(self, term)
|
||||
if is_data(term):
|
||||
self.value, self.length = term.items()[0]
|
||||
else:
|
||||
self.value, self.length = term
|
||||
|
||||
def __data__(self):
|
||||
if is_op(self.value, Variable) and is_literal(self.length):
|
||||
return {"left": {self.value.var: self.length.value}}
|
||||
else:
|
||||
return {"left": [self.value.__data__(), self.length.__data__()]}
|
||||
|
||||
def vars(self):
|
||||
return self.value.vars() | self.length.vars()
|
||||
|
||||
def map(self, map_):
|
||||
return self.lang[LeftOp([self.value.map(map_), self.length.map(map_)])]
|
||||
|
||||
def missing(self):
|
||||
return self.lang[
|
||||
OrOp([self.value.missing(), self.length.missing()])
|
||||
].partial_eval()
|
||||
|
||||
@simplified
|
||||
def partial_eval(self):
|
||||
value = self.lang[self.value].partial_eval()
|
||||
length = self.lang[self.length].partial_eval()
|
||||
max_length = LengthOp(value)
|
||||
|
||||
return self.lang[
|
||||
WhenOp(
|
||||
self.missing(),
|
||||
**{
|
||||
"else": BasicSubstringOp(
|
||||
[value, ZERO, MaxOp([ZERO, MinOp([length, max_length])])]
|
||||
)
|
||||
}
|
||||
)
|
||||
].partial_eval()
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions._utils import simplified
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.literal import Literal
|
||||
from jx_base.expressions.literal import is_literal
|
||||
from jx_base.expressions.null_op import NULL
|
||||
from jx_base.language import is_op
|
||||
from mo_future import is_text
|
||||
from mo_json import INTEGER
|
||||
|
||||
|
||||
class LengthOp(Expression):
|
||||
data_type = INTEGER
|
||||
|
||||
def __init__(self, term):
|
||||
Expression.__init__(self, [term])
|
||||
self.term = term
|
||||
|
||||
def __eq__(self, other):
|
||||
if is_op(other, LengthOp):
|
||||
return self.term == other.term
|
||||
|
||||
def __data__(self):
|
||||
return {"length": self.term.__data__()}
|
||||
|
||||
def vars(self):
|
||||
return self.term.vars()
|
||||
|
||||
def map(self, map_):
|
||||
return self.lang[LengthOp(self.term.map(map_))]
|
||||
|
||||
def missing(self):
|
||||
return self.term.missing()
|
||||
|
||||
@simplified
|
||||
def partial_eval(self):
|
||||
term = self.lang[self.term].partial_eval()
|
||||
if is_literal(term):
|
||||
if is_text(term.value):
|
||||
return self.lang[Literal(len(term.value))]
|
||||
else:
|
||||
return NULL
|
||||
else:
|
||||
return self.lang[LengthOp(term)]
|
||||
|
|
@ -0,0 +1,142 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions import _utils, expression
|
||||
from jx_base.expressions._utils import simplified, value2json
|
||||
from jx_base.expressions.expression import Expression
|
||||
from mo_dots import Null, is_data
|
||||
from mo_json import python_type_to_json_type
|
||||
|
||||
DateOp, FALSE, TRUE, NULL = [None]*4
|
||||
|
||||
class Literal(Expression):
|
||||
"""
|
||||
A literal JSON document
|
||||
"""
|
||||
|
||||
def __new__(cls, term):
|
||||
if term == None:
|
||||
return NULL
|
||||
if term is True:
|
||||
return TRUE
|
||||
if term is False:
|
||||
return FALSE
|
||||
if is_data(term) and term.get("date"):
|
||||
# SPECIAL CASE
|
||||
return cls.lang[DateOp(term.get("date"))]
|
||||
return object.__new__(cls)
|
||||
|
||||
def __init__(self, value):
|
||||
Expression.__init__(self, None)
|
||||
self.simplified = True
|
||||
self._value = value
|
||||
|
||||
@classmethod
|
||||
def define(cls, expr):
|
||||
return Literal(expr.get("literal"))
|
||||
|
||||
def __nonzero__(self):
|
||||
return True
|
||||
|
||||
def __eq__(self, other):
|
||||
if other == None:
|
||||
if self._value == None:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
elif self._value == None:
|
||||
return False
|
||||
|
||||
if is_literal(other):
|
||||
return (self._value == other._value) or (self.json == other.json)
|
||||
|
||||
def __data__(self):
|
||||
return {"literal": self.value}
|
||||
|
||||
@property
|
||||
def value(self):
|
||||
return self._value
|
||||
|
||||
@property
|
||||
def json(self):
|
||||
if self._value == "":
|
||||
self._json = '""'
|
||||
else:
|
||||
self._json = value2json(self._value)
|
||||
|
||||
return self._json
|
||||
|
||||
def vars(self):
|
||||
return set()
|
||||
|
||||
def map(self, map_):
|
||||
return self
|
||||
|
||||
def missing(self):
|
||||
if self._value in [None, Null]:
|
||||
return TRUE
|
||||
if self.value == "":
|
||||
return TRUE
|
||||
return FALSE
|
||||
|
||||
def __call__(self, row=None, rownum=None, rows=None):
|
||||
return self.value
|
||||
|
||||
def __unicode__(self):
|
||||
return self._json
|
||||
|
||||
def __str__(self):
|
||||
return str(self._json)
|
||||
|
||||
@property
|
||||
def type(self):
|
||||
return python_type_to_json_type[self._value.__class__]
|
||||
|
||||
@simplified
|
||||
def partial_eval(self):
|
||||
return self
|
||||
|
||||
def str(self):
|
||||
return str(self.value)
|
||||
|
||||
|
||||
ZERO = Literal(0)
|
||||
ONE = Literal(1)
|
||||
|
||||
|
||||
literal_op_ids = tuple()
|
||||
|
||||
|
||||
def register_literal(op):
|
||||
global literal_op_ids
|
||||
literal_op_ids = literal_op_ids+(op.get_id(),)
|
||||
|
||||
|
||||
def is_literal(l):
|
||||
try:
|
||||
return l.get_id() in literal_op_ids
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
_utils.Literal = Literal
|
||||
expression.Literal = Literal
|
||||
expression.is_literal=is_literal
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions.base_inequality_op import BaseInequalityOp
|
||||
|
||||
|
||||
class LtOp(BaseInequalityOp):
|
||||
op = "lt"
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions.base_inequality_op import BaseInequalityOp
|
||||
|
||||
|
||||
class LteOp(BaseInequalityOp):
|
||||
op = "lte"
|
||||
|
|
@ -0,0 +1,82 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions._utils import simplified
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.false_op import FALSE
|
||||
from jx_base.expressions.literal import Literal, is_literal
|
||||
from jx_base.expressions.null_op import NULL
|
||||
from mo_dots import is_many
|
||||
from mo_json import NUMBER
|
||||
from mo_math import MAX
|
||||
|
||||
|
||||
class MaxOp(Expression):
|
||||
data_type = NUMBER
|
||||
|
||||
def __init__(self, terms):
|
||||
Expression.__init__(self, terms)
|
||||
if terms == None:
|
||||
self.terms = []
|
||||
elif is_many(terms):
|
||||
self.terms = [t for t in terms if t != None]
|
||||
else:
|
||||
self.terms = [terms]
|
||||
|
||||
def __data__(self):
|
||||
return {"max": [t.__data__() for t in self.terms]}
|
||||
|
||||
def vars(self):
|
||||
output = set()
|
||||
for t in self.terms:
|
||||
output |= t.vars()
|
||||
return output
|
||||
|
||||
def map(self, map_):
|
||||
return self.lang[MaxOp([t.map(map_) for t in self.terms])]
|
||||
|
||||
def missing(self):
|
||||
return FALSE
|
||||
|
||||
@simplified
|
||||
def partial_eval(self):
|
||||
maximum = None
|
||||
terms = []
|
||||
for t in self.terms:
|
||||
simple = t.partial_eval()
|
||||
if simple is NULL:
|
||||
pass
|
||||
elif is_literal(simple):
|
||||
maximum = MAX([maximum, simple.value])
|
||||
else:
|
||||
terms.append(simple)
|
||||
if len(terms) == 0:
|
||||
if maximum == None:
|
||||
return NULL
|
||||
else:
|
||||
return Literal(maximum)
|
||||
else:
|
||||
if maximum == None:
|
||||
output = self.lang[MaxOp(terms)]
|
||||
else:
|
||||
output = self.lang[MaxOp([Literal(maximum)] + terms)]
|
||||
|
||||
return output
|
||||
|
|
@ -0,0 +1,85 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions._utils import simplified
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.false_op import FALSE
|
||||
from jx_base.expressions.literal import Literal
|
||||
from jx_base.expressions.literal import is_literal
|
||||
from jx_base.expressions.null_op import NULL
|
||||
from jx_base.expressions.null_op import NullOp
|
||||
from jx_base.language import is_op
|
||||
from mo_dots import is_many
|
||||
from mo_json import NUMBER
|
||||
from mo_math import MIN
|
||||
|
||||
|
||||
class MinOp(Expression):
|
||||
data_type = NUMBER
|
||||
|
||||
def __init__(self, terms):
|
||||
Expression.__init__(self, terms)
|
||||
if terms == None:
|
||||
self.terms = []
|
||||
elif is_many(terms):
|
||||
self.terms = terms
|
||||
else:
|
||||
self.terms = [terms]
|
||||
|
||||
def __data__(self):
|
||||
return {"min": [t.__data__() for t in self.terms]}
|
||||
|
||||
def vars(self):
|
||||
output = set()
|
||||
for t in self.terms:
|
||||
output |= t.vars()
|
||||
return output
|
||||
|
||||
def map(self, map_):
|
||||
return self.lang[MinOp([t.map(map_) for t in self.terms])]
|
||||
|
||||
def missing(self):
|
||||
return FALSE
|
||||
|
||||
@simplified
|
||||
def partial_eval(self):
|
||||
minimum = None
|
||||
terms = []
|
||||
for t in self.terms:
|
||||
simple = t.partial_eval()
|
||||
if is_op(simple, NullOp):
|
||||
pass
|
||||
elif is_literal(simple):
|
||||
minimum = MIN([minimum, simple.value])
|
||||
else:
|
||||
terms.append(simple)
|
||||
if len(terms) == 0:
|
||||
if minimum == None:
|
||||
return NULL
|
||||
else:
|
||||
return Literal(minimum)
|
||||
else:
|
||||
if minimum == None:
|
||||
output = self.lang[MinOp(terms)]
|
||||
else:
|
||||
output = self.lang[MinOp([Literal(minimum)] + terms)]
|
||||
|
||||
return output
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions import expression
|
||||
from jx_base.expressions._utils import simplified
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.false_op import FALSE
|
||||
from jx_base.expressions.true_op import TRUE
|
||||
from jx_base.language import is_op
|
||||
from mo_json import BOOLEAN
|
||||
|
||||
|
||||
class MissingOp(Expression):
|
||||
data_type = BOOLEAN
|
||||
|
||||
def __init__(self, term):
|
||||
Expression.__init__(self, term)
|
||||
self.expr = term
|
||||
|
||||
def __data__(self):
|
||||
return {"missing": self.expr.__data__()}
|
||||
|
||||
def __eq__(self, other):
|
||||
if not is_op(other, MissingOp):
|
||||
return False
|
||||
else:
|
||||
return self.expr == other.expr
|
||||
|
||||
def vars(self):
|
||||
return self.expr.vars()
|
||||
|
||||
def map(self, map_):
|
||||
return self.lang[MissingOp(self.expr.map(map_))]
|
||||
|
||||
def missing(self):
|
||||
return FALSE
|
||||
|
||||
def exists(self):
|
||||
return TRUE
|
||||
|
||||
@simplified
|
||||
def partial_eval(self):
|
||||
output = self.lang[self.expr].partial_eval().missing()
|
||||
if is_op(output, MissingOp):
|
||||
return output
|
||||
else:
|
||||
return output.partial_eval()
|
||||
|
||||
|
||||
expression.MissingOp = MissingOp
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions.base_binary_op import BaseBinaryOp
|
||||
|
||||
|
||||
class ModOp(BaseBinaryOp):
|
||||
op = "mod"
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions.base_multi_op import BaseMultiOp
|
||||
|
||||
|
||||
class MulOp(BaseMultiOp):
|
||||
op = "mul"
|
||||
|
|
@ -0,0 +1,71 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions import not_op
|
||||
from jx_base.expressions._utils import simplified
|
||||
from jx_base.expressions.eq_op import EqOp
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.false_op import FALSE
|
||||
from jx_base.expressions.literal import is_literal
|
||||
from jx_base.expressions.not_op import NotOp
|
||||
from jx_base.expressions.variable import Variable
|
||||
from jx_base.language import is_op
|
||||
from mo_dots import is_data, is_sequence
|
||||
from mo_json import BOOLEAN
|
||||
from mo_logs import Log
|
||||
|
||||
|
||||
class NeOp(Expression):
|
||||
has_simple_form = True
|
||||
data_type = BOOLEAN
|
||||
|
||||
def __init__(self, terms):
|
||||
Expression.__init__(self, terms)
|
||||
if is_sequence(terms):
|
||||
self.lhs, self.rhs = terms
|
||||
elif is_data(terms):
|
||||
self.rhs, self.lhs = terms.items()[0]
|
||||
else:
|
||||
Log.error("logic error")
|
||||
|
||||
def __data__(self):
|
||||
if is_op(self.lhs, Variable) and is_literal(self.rhs):
|
||||
return {"ne": {self.lhs.var, self.rhs.value}}
|
||||
else:
|
||||
return {"ne": [self.lhs.__data__(), self.rhs.__data__()]}
|
||||
|
||||
def vars(self):
|
||||
return self.lhs.vars() | self.rhs.vars()
|
||||
|
||||
def map(self, map_):
|
||||
return self.lang[NeOp([self.lhs.map(map_), self.rhs.map(map_)])]
|
||||
|
||||
def missing(self):
|
||||
return (
|
||||
FALSE
|
||||
) # USING THE decisive EQUAILTY https://github.com/mozilla/jx-sqlite/blob/master/docs/Logical%20Equality.md#definitions
|
||||
|
||||
@simplified
|
||||
def partial_eval(self):
|
||||
output = self.lang[NotOp(EqOp([self.lhs, self.rhs]))].partial_eval()
|
||||
return output
|
||||
|
||||
not_op.NeOp = NeOp
|
||||
|
|
@ -0,0 +1,83 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions._utils import simplified
|
||||
from jx_base.expressions.basic_substring_op import BasicSubstringOp
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.length_op import LengthOp
|
||||
from jx_base.expressions.literal import ZERO
|
||||
from jx_base.expressions.literal import is_literal
|
||||
from jx_base.expressions.max_op import MaxOp
|
||||
from jx_base.expressions.min_op import MinOp
|
||||
from jx_base.expressions.or_op import OrOp
|
||||
from jx_base.expressions.variable import Variable
|
||||
from jx_base.expressions.when_op import WhenOp
|
||||
from jx_base.language import is_op
|
||||
from mo_dots import is_data
|
||||
from mo_json import STRING
|
||||
|
||||
|
||||
class NotLeftOp(Expression):
|
||||
has_simple_form = True
|
||||
data_type = STRING
|
||||
|
||||
def __init__(self, term):
|
||||
Expression.__init__(self, term)
|
||||
if is_data(term):
|
||||
self.value, self.length = term.items()[0]
|
||||
else:
|
||||
self.value, self.length = term
|
||||
|
||||
def __data__(self):
|
||||
if is_op(self.value, Variable) and is_literal(self.length):
|
||||
return {"not_left": {self.value.var: self.length.value}}
|
||||
else:
|
||||
return {"not_left": [self.value.__data__(), self.length.__data__()]}
|
||||
|
||||
def vars(self):
|
||||
return self.value.vars() | self.length.vars()
|
||||
|
||||
def map(self, map_):
|
||||
return self.lang[NotLeftOp([self.value.map(map_), self.length.map(map_)])]
|
||||
|
||||
def missing(self):
|
||||
return self.lang[OrOp([self.value.missing(), self.length.missing()])]
|
||||
|
||||
@simplified
|
||||
def partial_eval(self):
|
||||
value = self.lang[self.value].partial_eval()
|
||||
length = self.length.partial_eval()
|
||||
|
||||
if length is ZERO:
|
||||
return value
|
||||
|
||||
max_length = LengthOp(value)
|
||||
output = self.lang[
|
||||
WhenOp(
|
||||
self.missing(),
|
||||
**{
|
||||
"else": BasicSubstringOp(
|
||||
[value, MaxOp([ZERO, MinOp([length, max_length])]), max_length]
|
||||
)
|
||||
}
|
||||
)
|
||||
].partial_eval()
|
||||
return output
|
||||
|
|
@ -0,0 +1,126 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions import and_op, exists_op, expression
|
||||
from jx_base.expressions._utils import simplified
|
||||
from jx_base.expressions.and_op import AndOp
|
||||
from jx_base.expressions.basic_index_of_op import BasicIndexOfOp
|
||||
from jx_base.expressions.basic_substring_op import BasicSubstringOp
|
||||
from jx_base.expressions.eq_op import EqOp
|
||||
from jx_base.expressions.exists_op import ExistsOp
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.false_op import FALSE
|
||||
from jx_base.expressions.literal import is_literal
|
||||
from jx_base.expressions.missing_op import MissingOp
|
||||
from jx_base.expressions.null_op import NULL
|
||||
from jx_base.expressions.or_op import OrOp
|
||||
from jx_base.expressions.true_op import TRUE
|
||||
from jx_base.language import is_op
|
||||
from mo_json import BOOLEAN
|
||||
from mo_logs import Log
|
||||
|
||||
CaseOp = None
|
||||
NeOp = None
|
||||
WhenOp = None
|
||||
|
||||
class NotOp(Expression):
|
||||
data_type = BOOLEAN
|
||||
|
||||
def __init__(self, term):
|
||||
Expression.__init__(self, term)
|
||||
self.term = term
|
||||
|
||||
def __data__(self):
|
||||
return {"not": self.term.__data__()}
|
||||
|
||||
def __eq__(self, other):
|
||||
if not is_op(other, NotOp):
|
||||
return False
|
||||
return self.term == other.term
|
||||
|
||||
def vars(self):
|
||||
return self.term.vars()
|
||||
|
||||
def map(self, map_):
|
||||
return self.lang[NotOp(self.term.map(map_))]
|
||||
|
||||
def missing(self):
|
||||
return self.term.missing()
|
||||
|
||||
@simplified
|
||||
def partial_eval(self):
|
||||
def inverse(term):
|
||||
if term is TRUE:
|
||||
return FALSE
|
||||
elif term is FALSE:
|
||||
return TRUE
|
||||
elif term is NULL:
|
||||
return TRUE
|
||||
elif is_literal(term):
|
||||
Log.error("`not` operator expects a Boolean term")
|
||||
elif is_op(term, WhenOp):
|
||||
output = self.lang[
|
||||
WhenOp(
|
||||
term.when,
|
||||
**{"then": inverse(term.then), "else": inverse(term.els_)}
|
||||
)
|
||||
].partial_eval()
|
||||
elif is_op(term, CaseOp): # REWRITING
|
||||
output = self.lang[
|
||||
CaseOp(
|
||||
[
|
||||
WhenOp(w.when, **{"then": inverse(w.then)})
|
||||
if is_op(w, WhenOp)
|
||||
else inverse(w)
|
||||
for w in term.whens
|
||||
]
|
||||
)
|
||||
].partial_eval()
|
||||
elif is_op(term, AndOp):
|
||||
output = self.lang[
|
||||
OrOp([inverse(t) for t in term.terms])
|
||||
].partial_eval()
|
||||
elif is_op(term, OrOp):
|
||||
output = self.lang[
|
||||
AndOp([inverse(t) for t in term.terms])
|
||||
].partial_eval()
|
||||
elif is_op(term, MissingOp):
|
||||
output = self.lang[NotOp(term.expr.missing())]
|
||||
elif is_op(term, ExistsOp):
|
||||
output = term.field.missing().partial_eval()
|
||||
elif is_op(term, NotOp):
|
||||
output = self.lang[term.term].partial_eval()
|
||||
elif is_op(term, NeOp):
|
||||
output = self.lang[EqOp([term.lhs, term.rhs])].partial_eval()
|
||||
elif is_op(term, BasicIndexOfOp) or is_op(term, BasicSubstringOp):
|
||||
return FALSE
|
||||
else:
|
||||
output = self.lang[NotOp(term)]
|
||||
|
||||
return output
|
||||
|
||||
output = inverse(self.lang[self.term].partial_eval())
|
||||
return output
|
||||
|
||||
|
||||
and_op.NotOp = NotOp
|
||||
exists_op.NotOp = NotOp
|
||||
expression.NotOp =NotOp
|
||||
|
|
@ -0,0 +1,81 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions._utils import simplified
|
||||
from jx_base.expressions.basic_substring_op import BasicSubstringOp
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.length_op import LengthOp
|
||||
from jx_base.expressions.literal import ZERO
|
||||
from jx_base.expressions.literal import is_literal
|
||||
from jx_base.expressions.max_op import MaxOp
|
||||
from jx_base.expressions.min_op import MinOp
|
||||
from jx_base.expressions.or_op import OrOp
|
||||
from jx_base.expressions.sub_op import SubOp
|
||||
from jx_base.expressions.variable import Variable
|
||||
from jx_base.expressions.when_op import WhenOp
|
||||
from jx_base.language import is_op
|
||||
from mo_dots import is_data
|
||||
from mo_json import STRING
|
||||
|
||||
|
||||
class NotRightOp(Expression):
|
||||
has_simple_form = True
|
||||
data_type = STRING
|
||||
|
||||
def __init__(self, term):
|
||||
Expression.__init__(self, term)
|
||||
if is_data(term):
|
||||
self.value, self.length = term.items()[0]
|
||||
else:
|
||||
self.value, self.length = term
|
||||
|
||||
def __data__(self):
|
||||
if is_op(self.value, Variable) and is_literal(self.length):
|
||||
return {"not_right": {self.value.var: self.length.value}}
|
||||
else:
|
||||
return {"not_right": [self.value.__data__(), self.length.__data__()]}
|
||||
|
||||
def vars(self):
|
||||
return self.value.vars() | self.length.vars()
|
||||
|
||||
def map(self, map_):
|
||||
return self.lang[NotRightOp([self.value.map(map_), self.length.map(map_)])]
|
||||
|
||||
def missing(self):
|
||||
return self.lang[OrOp([self.value.missing(), self.length.missing()])]
|
||||
|
||||
@simplified
|
||||
def partial_eval(self):
|
||||
value = self.lang[self.value].partial_eval()
|
||||
length = self.length.partial_eval()
|
||||
|
||||
if length is ZERO:
|
||||
return value
|
||||
|
||||
max_length = LengthOp(value)
|
||||
part = BasicSubstringOp(
|
||||
[
|
||||
value,
|
||||
ZERO,
|
||||
MaxOp([ZERO, MinOp([max_length, SubOp([max_length, length])])]),
|
||||
]
|
||||
)
|
||||
return self.lang[WhenOp(self.missing(), **{"else": part})].partial_eval()
|
||||
|
|
@ -0,0 +1,113 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions import literal, _utils, expression
|
||||
from jx_base.expressions.false_op import FALSE
|
||||
from jx_base.expressions.literal import Literal
|
||||
from jx_base.expressions.true_op import TRUE
|
||||
from jx_base.language import TYPE_ORDER
|
||||
from mo_dots import Null
|
||||
from mo_json import IS_NULL, OBJECT
|
||||
from mo_logs import Log
|
||||
|
||||
|
||||
class NullOp(Literal):
|
||||
"""
|
||||
FOR USE WHEN EVERYTHING IS EXPECTED TO BE AN Expression
|
||||
USE IT TO EXPECT A NULL VALUE IN assertAlmostEqual
|
||||
"""
|
||||
|
||||
data_type = OBJECT
|
||||
|
||||
@classmethod
|
||||
def define(cls, expr):
|
||||
return NULL
|
||||
|
||||
def __new__(cls, *args, **kwargs):
|
||||
return object.__new__(cls, *args, **kwargs)
|
||||
|
||||
def __init__(self, op=None, term=None):
|
||||
Literal.__init__(self, None)
|
||||
|
||||
def __nonzero__(self):
|
||||
return True
|
||||
|
||||
def __eq__(self, other):
|
||||
return other is NULL
|
||||
|
||||
def __gt__(self, other):
|
||||
return False
|
||||
|
||||
def __lt__(self, other):
|
||||
return False
|
||||
|
||||
def __ge__(self, other):
|
||||
if other == None:
|
||||
return True
|
||||
return False
|
||||
|
||||
def __le__(self, other):
|
||||
if other == None:
|
||||
return True
|
||||
return False
|
||||
|
||||
def __data__(self):
|
||||
return {"null": {}}
|
||||
|
||||
def vars(self):
|
||||
return set()
|
||||
|
||||
def map(self, map_):
|
||||
return self
|
||||
|
||||
def missing(self):
|
||||
return TRUE
|
||||
|
||||
def exists(self):
|
||||
return FALSE
|
||||
|
||||
def __call__(self, row=None, rownum=None, rows=None):
|
||||
return Null
|
||||
|
||||
def __unicode__(self):
|
||||
return "null"
|
||||
|
||||
def __str__(self):
|
||||
return b"null"
|
||||
|
||||
@property
|
||||
def type(self):
|
||||
return IS_NULL
|
||||
|
||||
def __hash__(self):
|
||||
return id(None)
|
||||
|
||||
def __bool__(self):
|
||||
Log.error("Detecting truthiness of NullOp is too confusing to be allowed")
|
||||
|
||||
|
||||
NULL = NullOp()
|
||||
TYPE_ORDER[NullOp] = 9
|
||||
TYPE_ORDER[NULL] = 9
|
||||
|
||||
literal.NULL = NULL
|
||||
_utils.NULL = NULL
|
||||
expression.NULL=NULL
|
||||
|
|
@ -0,0 +1,97 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions._utils import simplified
|
||||
from jx_base.expressions.case_op import CaseOp
|
||||
from jx_base.expressions.coalesce_op import CoalesceOp
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.false_op import FALSE
|
||||
from jx_base.expressions.first_op import FirstOp
|
||||
from jx_base.expressions.literal import Literal, ZERO, ONE
|
||||
from jx_base.expressions.literal import is_literal
|
||||
from jx_base.expressions.null_op import NULL
|
||||
from jx_base.expressions.true_op import TRUE
|
||||
from jx_base.expressions.when_op import WhenOp
|
||||
from jx_base.language import is_op
|
||||
from mo_future import text
|
||||
from mo_json import NUMBER
|
||||
from mo_logs import Log
|
||||
from mo_times import Date
|
||||
|
||||
|
||||
class NumberOp(Expression):
|
||||
data_type = NUMBER
|
||||
|
||||
def __init__(self, term):
|
||||
Expression.__init__(self, [term])
|
||||
self.term = term
|
||||
|
||||
def __data__(self):
|
||||
return {"number": self.term.__data__()}
|
||||
|
||||
def vars(self):
|
||||
return self.term.vars()
|
||||
|
||||
def map(self, map_):
|
||||
return self.lang[NumberOp(self.term.map(map_))]
|
||||
|
||||
def missing(self):
|
||||
return self.term.missing()
|
||||
|
||||
@simplified
|
||||
def partial_eval(self):
|
||||
term = self.lang[FirstOp(self.term)].partial_eval()
|
||||
|
||||
if is_literal(term):
|
||||
if term is NULL:
|
||||
return NULL
|
||||
elif term is FALSE:
|
||||
return ZERO
|
||||
elif term is TRUE:
|
||||
return ONE
|
||||
|
||||
v = term.value
|
||||
if isinstance(v, (text, Date)):
|
||||
return self.lang[Literal(float(v))]
|
||||
elif isinstance(v, (int, float)):
|
||||
return term
|
||||
else:
|
||||
Log.error("can not convert {{value|json}} to number", value=term.value)
|
||||
elif is_op(term, CaseOp): # REWRITING
|
||||
return self.lang[
|
||||
CaseOp(
|
||||
[
|
||||
WhenOp(t.when, **{"then": NumberOp(t.then)})
|
||||
for t in term.whens[:-1]
|
||||
]
|
||||
+ [NumberOp(term.whens[-1])]
|
||||
)
|
||||
].partial_eval()
|
||||
elif is_op(term, WhenOp): # REWRITING
|
||||
return self.lang[
|
||||
WhenOp(
|
||||
term.when,
|
||||
**{"then": NumberOp(term.then), "else": NumberOp(term.els_)}
|
||||
)
|
||||
].partial_eval()
|
||||
elif is_op(term, CoalesceOp):
|
||||
return self.lang[CoalesceOp([NumberOp(t) for t in term.terms])]
|
||||
return self.lang[NumberOp(term)]
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions.expression import Expression
|
||||
from mo_future import text
|
||||
from mo_logs import Log
|
||||
from mo_math import is_integer
|
||||
|
||||
|
||||
class OffsetOp(Expression):
|
||||
"""
|
||||
OFFSET INDEX INTO A TUPLE
|
||||
"""
|
||||
|
||||
def __init__(self, var):
|
||||
Expression.__init__(self, None)
|
||||
if not is_integer(var):
|
||||
Log.error("Expecting an integer")
|
||||
self.var = var
|
||||
|
||||
def __call__(self, row, rownum=None, rows=None):
|
||||
try:
|
||||
return row[self.var]
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def __data__(self):
|
||||
return {"offset": self.var}
|
||||
|
||||
def vars(self):
|
||||
return {}
|
||||
|
||||
def __hash__(self):
|
||||
return self.var.__hash__()
|
||||
|
||||
def __eq__(self, other):
|
||||
return self.var == other
|
||||
|
||||
def __unicode__(self):
|
||||
return text(self.var)
|
||||
|
||||
def __str__(self):
|
||||
return str(self.var)
|
||||
|
|
@ -0,0 +1,99 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions import and_op
|
||||
from jx_base.expressions._utils import simplified
|
||||
from jx_base.expressions.and_op import AndOp
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.false_op import FALSE
|
||||
from jx_base.expressions.true_op import TRUE
|
||||
from jx_base.language import is_op
|
||||
from mo_json import BOOLEAN
|
||||
|
||||
|
||||
class OrOp(Expression):
|
||||
data_type = BOOLEAN
|
||||
|
||||
def __init__(self, terms):
|
||||
Expression.__init__(self, terms)
|
||||
self.terms = terms
|
||||
|
||||
def __data__(self):
|
||||
return {"or": [t.__data__() for t in self.terms]}
|
||||
|
||||
def vars(self):
|
||||
output = set()
|
||||
for t in self.terms:
|
||||
output |= t.vars()
|
||||
return output
|
||||
|
||||
def map(self, map_):
|
||||
return self.lang[OrOp([t.map(map_) for t in self.terms])]
|
||||
|
||||
def missing(self):
|
||||
return FALSE
|
||||
|
||||
def __call__(self, row=None, rownum=None, rows=None):
|
||||
return any(t(row, rownum, rows) for t in self.terms)
|
||||
|
||||
def __eq__(self, other):
|
||||
if not is_op(other, OrOp):
|
||||
return False
|
||||
if len(self.terms) != len(other.terms):
|
||||
return False
|
||||
return all(t == u for t, u in zip(self.terms, other.terms))
|
||||
|
||||
@simplified
|
||||
def partial_eval(self):
|
||||
terms = []
|
||||
ands = []
|
||||
for t in self.terms:
|
||||
simple = self.lang[t].partial_eval()
|
||||
if simple.type != BOOLEAN:
|
||||
simple = simple.exists()
|
||||
|
||||
if simple is TRUE:
|
||||
return TRUE
|
||||
elif simple is FALSE:
|
||||
pass
|
||||
elif is_op(simple, OrOp):
|
||||
terms.extend([tt for tt in simple.terms if tt not in terms])
|
||||
elif is_op(simple, AndOp):
|
||||
ands.append(simple)
|
||||
elif simple not in terms:
|
||||
terms.append(simple)
|
||||
|
||||
if ands: # REMOVE TERMS THAT ARE MORE RESTRICTIVE THAN OTHERS
|
||||
for a in ands:
|
||||
for tt in a.terms:
|
||||
if tt in terms:
|
||||
break
|
||||
else:
|
||||
terms.append(a)
|
||||
|
||||
if len(terms) == 0:
|
||||
return FALSE
|
||||
if len(terms) == 1:
|
||||
return terms[0]
|
||||
return self.lang[OrOp(terms)]
|
||||
|
||||
|
||||
and_op.OrOp = OrOp
|
||||
|
|
@ -0,0 +1,88 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions._utils import simplified
|
||||
from jx_base.expressions.basic_starts_with_op import BasicStartsWithOp
|
||||
from jx_base.expressions.case_op import CaseOp
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.false_op import FALSE
|
||||
from jx_base.expressions.literal import is_literal
|
||||
from jx_base.expressions.null_op import NULL
|
||||
from jx_base.expressions.true_op import TRUE
|
||||
from jx_base.expressions.variable import Variable
|
||||
from jx_base.expressions.when_op import WhenOp
|
||||
from jx_base.language import is_op
|
||||
from mo_dots import is_data
|
||||
from mo_json import BOOLEAN
|
||||
|
||||
|
||||
class PrefixOp(Expression):
|
||||
has_simple_form = True
|
||||
data_type = BOOLEAN
|
||||
|
||||
def __init__(self, term):
|
||||
Expression.__init__(self, term)
|
||||
if not term:
|
||||
self.expr = NULL
|
||||
self.prefix = NULL
|
||||
elif is_data(term):
|
||||
self.expr, self.prefix = term.items()[0]
|
||||
else:
|
||||
self.expr, self.prefix = term
|
||||
|
||||
def __data__(self):
|
||||
if not self.expr:
|
||||
return {"prefix": {}}
|
||||
elif is_op(self.expr, Variable) and is_literal(self.prefix):
|
||||
return {"prefix": {self.expr.var: self.prefix.value}}
|
||||
else:
|
||||
return {"prefix": [self.expr.__data__(), self.prefix.__data__()]}
|
||||
|
||||
def vars(self):
|
||||
if self.expr is NULL:
|
||||
return set()
|
||||
return self.expr.vars() | self.prefix.vars()
|
||||
|
||||
def map(self, map_):
|
||||
if not self.expr:
|
||||
return self
|
||||
else:
|
||||
return self.lang[PrefixOp([self.expr.map(map_), self.prefix.map(map_)])]
|
||||
|
||||
def missing(self):
|
||||
return FALSE
|
||||
|
||||
@simplified
|
||||
def partial_eval(self):
|
||||
return self.lang[
|
||||
CaseOp(
|
||||
[
|
||||
WhenOp(self.prefix.missing(), then=TRUE),
|
||||
WhenOp(self.expr.missing(), then=FALSE),
|
||||
BasicStartsWithOp([self.expr, self.prefix]),
|
||||
]
|
||||
)
|
||||
].partial_eval()
|
||||
|
||||
def __eq__(self, other):
|
||||
if not is_op(other, PrefixOp):
|
||||
return False
|
||||
return self.expr == other.expr and self.prefix == other.prefix
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions.expression import Expression
|
||||
|
||||
|
||||
class PythonScript(Expression):
|
||||
"""
|
||||
REPRESENT A Python SCRIPT
|
||||
"""
|
||||
|
||||
pass
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions.expression import Expression
|
||||
|
||||
|
||||
class QueryOp(Expression):
|
||||
pass
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions._utils import operators
|
||||
from jx_base.expressions.and_op import AndOp
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.literal import Literal
|
||||
from mo_json import BOOLEAN
|
||||
from mo_logs import Log
|
||||
|
||||
|
||||
class RangeOp(Expression):
|
||||
has_simple_form = True
|
||||
data_type = BOOLEAN
|
||||
|
||||
def __new__(cls, term, *args):
|
||||
Expression.__new__(cls, *args)
|
||||
field, comparisons = term # comparisons IS A Literal()
|
||||
return cls.lang[
|
||||
AndOp(
|
||||
[
|
||||
getattr(cls.lang, operators[op])([field, Literal(value)])
|
||||
for op, value in comparisons.value.items()
|
||||
]
|
||||
)
|
||||
]
|
||||
|
||||
def __init__(self, term):
|
||||
Log.error("Should never happen!")
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.false_op import FALSE
|
||||
from jx_base.expressions.true_op import TRUE
|
||||
from mo_json import BOOLEAN
|
||||
|
||||
|
||||
class RegExpOp(Expression):
|
||||
has_simple_form = True
|
||||
data_type = BOOLEAN
|
||||
|
||||
def __init__(self, terms):
|
||||
Expression.__init__(self, terms)
|
||||
self.var, self.pattern = terms
|
||||
|
||||
def __data__(self):
|
||||
return {"regexp": {self.var.var: self.pattern}}
|
||||
|
||||
def vars(self):
|
||||
return {self.var}
|
||||
|
||||
def map(self, map_):
|
||||
return self.lang[RegExpOp([self.var.map(map_), self.pattern])]
|
||||
|
||||
def missing(self):
|
||||
return FALSE
|
||||
|
||||
def exists(self):
|
||||
return TRUE
|
||||
|
|
@ -0,0 +1,89 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions._utils import simplified
|
||||
from jx_base.expressions.basic_substring_op import BasicSubstringOp
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.length_op import LengthOp
|
||||
from jx_base.expressions.literal import ZERO
|
||||
from jx_base.expressions.literal import is_literal
|
||||
from jx_base.expressions.max_op import MaxOp
|
||||
from jx_base.expressions.min_op import MinOp
|
||||
from jx_base.expressions.or_op import OrOp
|
||||
from jx_base.expressions.sub_op import SubOp
|
||||
from jx_base.expressions.variable import Variable
|
||||
from jx_base.expressions.when_op import WhenOp
|
||||
from jx_base.language import is_op
|
||||
from mo_dots import is_data
|
||||
from mo_json import STRING
|
||||
|
||||
|
||||
class RightOp(Expression):
|
||||
has_simple_form = True
|
||||
data_type = STRING
|
||||
|
||||
def __init__(self, term):
|
||||
Expression.__init__(self, term)
|
||||
if is_data(term):
|
||||
self.value, self.length = term.items()[0]
|
||||
else:
|
||||
self.value, self.length = term
|
||||
|
||||
if is_literal(self.value):
|
||||
Log.note("")
|
||||
|
||||
def __data__(self):
|
||||
if is_op(self.value, Variable) and is_literal(self.length):
|
||||
return {"right": {self.value.var: self.length.value}}
|
||||
else:
|
||||
return {"right": [self.value.__data__(), self.length.__data__()]}
|
||||
|
||||
def vars(self):
|
||||
return self.value.vars() | self.length.vars()
|
||||
|
||||
def map(self, map_):
|
||||
return self.lang[RightOp([self.value.map(map_), self.length.map(map_)])]
|
||||
|
||||
def missing(self):
|
||||
return self.lang[OrOp([self.value.missing(), self.length.missing()])]
|
||||
|
||||
@simplified
|
||||
def partial_eval(self):
|
||||
value = self.lang[self.value].partial_eval()
|
||||
length = self.lang[self.length].partial_eval()
|
||||
max_length = LengthOp(value)
|
||||
|
||||
return self.lang[
|
||||
WhenOp(
|
||||
self.missing(),
|
||||
**{
|
||||
"else": BasicSubstringOp(
|
||||
[
|
||||
value,
|
||||
MaxOp(
|
||||
[ZERO, MinOp([max_length, SubOp([max_length, length])])]
|
||||
),
|
||||
max_length,
|
||||
]
|
||||
)
|
||||
}
|
||||
)
|
||||
].partial_eval()
|
||||
|
|
@ -0,0 +1,56 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.literal import Literal
|
||||
from jx_base.expressions.literal import is_literal
|
||||
from jx_base.expressions.variable import Variable
|
||||
from jx_base.language import is_op
|
||||
from mo_logs import Log
|
||||
|
||||
|
||||
class RowsOp(Expression):
|
||||
has_simple_form = True
|
||||
|
||||
def __init__(self, term):
|
||||
Expression.__init__(self, term)
|
||||
self.var, self.offset = term
|
||||
if is_op(self.var, Variable):
|
||||
if is_op(self.var, Variable) and not any(
|
||||
self.var.var.startswith(p) for p in ["row.", "rows.", "rownum"]
|
||||
): # VARIABLES ARE INTERPRETED LITERALLY
|
||||
self.var = Literal(self.var.var)
|
||||
else:
|
||||
Log.error("can not handle")
|
||||
else:
|
||||
Log.error("can not handle")
|
||||
|
||||
def __data__(self):
|
||||
if is_literal(self.var) and is_literal(self.offset):
|
||||
return {"rows": {self.var.json, self.offset.value}}
|
||||
else:
|
||||
return {"rows": [self.var.__data__(), self.offset.__data__()]}
|
||||
|
||||
def vars(self):
|
||||
return self.var.vars() | self.offset.vars() | {"rows", "rownum"}
|
||||
|
||||
def map(self, map_):
|
||||
return self.lang[RowsOp([self.var.map(map_), self.offset.map(map_)])]
|
||||
|
|
@ -0,0 +1,62 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions.expression import Expression
|
||||
from mo_future import is_text
|
||||
from mo_json import OBJECT
|
||||
from mo_logs import Log
|
||||
|
||||
|
||||
class ScriptOp(Expression):
|
||||
"""
|
||||
ONLY FOR WHEN YOU TRUST THE SCRIPT SOURCE
|
||||
"""
|
||||
|
||||
def __init__(self, script, data_type=OBJECT):
|
||||
Expression.__init__(self, None)
|
||||
if not is_text(script):
|
||||
Log.error("expecting text of a script")
|
||||
self.simplified = True
|
||||
self.script = script
|
||||
self.data_type = data_type
|
||||
|
||||
@classmethod
|
||||
def define(cls, expr):
|
||||
if ALLOW_SCRIPTING:
|
||||
Log.warning(
|
||||
"Scripting has been activated: This has known security holes!!\nscript = {{script|quote}}",
|
||||
script=expr.script.term,
|
||||
)
|
||||
return cls.lang[ScriptOp(expr.script)]
|
||||
else:
|
||||
Log.error("scripting is disabled")
|
||||
|
||||
def vars(self):
|
||||
return set()
|
||||
|
||||
def map(self, map_):
|
||||
return self
|
||||
|
||||
def __unicode__(self):
|
||||
return self.script
|
||||
|
||||
def __str__(self):
|
||||
return str(self.script)
|
||||
|
|
@ -0,0 +1,91 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions.expression import jx_expression, Expression, _jx_expression
|
||||
from jx_base.utils import is_variable_name
|
||||
from mo_dots import wrap, is_container
|
||||
from mo_future import is_text
|
||||
from mo_logs import Log
|
||||
from mo_math import UNION
|
||||
|
||||
|
||||
class SelectOp(Expression):
|
||||
has_simple_form = True
|
||||
|
||||
def __init__(self, terms):
|
||||
"""
|
||||
:param terms: list OF {"name":name, "value":value} DESCRIPTORS
|
||||
"""
|
||||
self.terms = terms
|
||||
|
||||
@classmethod
|
||||
def define(cls, expr):
|
||||
expr = wrap(expr)
|
||||
term = expr.select
|
||||
terms = []
|
||||
if not is_container(term):
|
||||
raise Log.error("Expecting a list")
|
||||
for t in term:
|
||||
if is_text(t):
|
||||
if not is_variable_name(t):
|
||||
Log.error(
|
||||
"expecting {{value}} a simple dot-delimited path name", value=t
|
||||
)
|
||||
terms.append({"name": t, "value": _jx_expression(t, cls.lang)})
|
||||
elif t.name == None:
|
||||
if t.value == None:
|
||||
Log.error(
|
||||
"expecting select parameters to have name and value properties"
|
||||
)
|
||||
elif is_text(t.value):
|
||||
if not is_variable_name(t):
|
||||
Log.error(
|
||||
"expecting {{value}} a simple dot-delimited path name",
|
||||
value=t.value,
|
||||
)
|
||||
else:
|
||||
terms.append(
|
||||
{
|
||||
"name": t.value,
|
||||
"value": _jx_expression(t.value, cls.lang),
|
||||
}
|
||||
)
|
||||
else:
|
||||
Log.error("expecting a name property")
|
||||
else:
|
||||
terms.append({"name": t.name, "value": jx_expression(t.value)})
|
||||
return cls.lang[SelectOp(terms)]
|
||||
|
||||
def __data__(self):
|
||||
return {
|
||||
"select": [
|
||||
{"name": t.name.__data__(), "value": t.value.__data__()}
|
||||
for t in self.terms
|
||||
]
|
||||
}
|
||||
|
||||
def vars(self):
|
||||
return UNION(t.value for t in self.terms)
|
||||
|
||||
def map(self, map_):
|
||||
return SelectOp(
|
||||
[{"name": t.name, "value": t.value.map(map_)} for t in self.terms]
|
||||
)
|
||||
|
|
@ -0,0 +1,84 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions.and_op import AndOp
|
||||
from jx_base.expressions.eq_op import EqOp
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.find_op import FindOp
|
||||
from jx_base.expressions.literal import Literal
|
||||
from jx_base.expressions.literal import is_literal
|
||||
from jx_base.expressions.or_op import OrOp
|
||||
from jx_base.expressions.script_op import ScriptOp
|
||||
from jx_base.expressions.true_op import TRUE
|
||||
from jx_base.expressions.variable import Variable
|
||||
from jx_base.language import is_op
|
||||
|
||||
|
||||
class SplitOp(Expression):
|
||||
has_simple_form = True
|
||||
|
||||
def __init__(self, term, **kwargs):
|
||||
Expression.__init__(self, term)
|
||||
self.value, self.find = term
|
||||
|
||||
def __data__(self):
|
||||
if is_op(self.value, Variable) and is_literal(self.find):
|
||||
return {"split": {self.value.var, self.find.value}}
|
||||
else:
|
||||
return {"split": [self.value.__data__(), self.find.__data__()]}
|
||||
|
||||
def vars(self):
|
||||
return (
|
||||
self.value.vars()
|
||||
| self.find.vars()
|
||||
| self.default.vars()
|
||||
| self.start.vars()
|
||||
)
|
||||
|
||||
def map(self, map_):
|
||||
return FindOp(
|
||||
[self.value.map(map_), self.find.map(map_)],
|
||||
start=self.start.map(map_),
|
||||
default=self.default.map(map_),
|
||||
)
|
||||
|
||||
def missing(self):
|
||||
v = self.value.to_es_script(not_null=True)
|
||||
find = self.find.to_es_script(not_null=True)
|
||||
index = v + ".indexOf(" + find + ", " + self.start.to_es_script() + ")"
|
||||
|
||||
return self.lang[
|
||||
AndOp(
|
||||
[
|
||||
self.default.missing(),
|
||||
OrOp(
|
||||
[
|
||||
self.value.missing(),
|
||||
self.find.missing(),
|
||||
EqOp([ScriptOp(index), Literal(-1)]),
|
||||
]
|
||||
),
|
||||
]
|
||||
)
|
||||
]
|
||||
|
||||
def exists(self):
|
||||
return TRUE
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions.eq_op import EqOp
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.false_op import FALSE
|
||||
from jx_base.language import is_op
|
||||
from mo_json import BOOLEAN
|
||||
|
||||
|
||||
class SqlEqOp(Expression):
|
||||
data_type = BOOLEAN
|
||||
|
||||
def __init__(self, terms):
|
||||
Expression.__init__(self, terms)
|
||||
self.lhs, self.rhs = terms
|
||||
|
||||
def __data__(self):
|
||||
return {"sql.eq": [self.lhs.__data__(), self.rhs.__data__()]}
|
||||
|
||||
def missing(self):
|
||||
return FALSE
|
||||
|
||||
def __eq__(self, other):
|
||||
if not is_op(other, EqOp):
|
||||
return False
|
||||
return self.lhs == other.lhs and self.rhs == other.rhs
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.false_op import FALSE
|
||||
from mo_json import INTEGER
|
||||
|
||||
|
||||
class SqlInstrOp(Expression):
|
||||
data_type = INTEGER
|
||||
|
||||
def __init__(self, params):
|
||||
Expression.__init__(self, params)
|
||||
self.value, self.find = params
|
||||
|
||||
def __data__(self):
|
||||
return {"sql.instr": [self.value.__data__(), self.find.__data__()]}
|
||||
|
||||
def vars(self):
|
||||
return self.value.vars() | self.find.vars()
|
||||
|
||||
def missing(self):
|
||||
return FALSE
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions.expression import Expression
|
||||
|
||||
|
||||
class SQLScript(Expression):
|
||||
"""
|
||||
REPRESENT A SQL SCRIPT
|
||||
"""
|
||||
|
||||
pass
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
# encoding: utf-8
|
||||
#
|
||||
#
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http:# mozilla.org/MPL/2.0/.
|
||||
#
|
||||
# Contact: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
|
||||
"""
|
||||
# NOTE:
|
||||
|
||||
THE self.lang[operator] PATTERN IS CASTING NEW OPERATORS TO OWN LANGUAGE;
|
||||
KEEPING Python AS# Python, ES FILTERS AS ES FILTERS, AND Painless AS
|
||||
Painless. WE COULD COPY partial_eval(), AND OTHERS, TO THIER RESPECTIVE
|
||||
LANGUAGE, BUT WE KEEP CODE HERE SO THERE IS LESS OF IT
|
||||
|
||||
"""
|
||||
from __future__ import absolute_import, division, unicode_literals
|
||||
|
||||
from jx_base.expressions.expression import Expression
|
||||
from jx_base.expressions.false_op import FALSE
|
||||
from mo_json import INTEGER
|
||||
|
||||
|
||||
class SqlSubstrOp(Expression):
|
||||
data_type = INTEGER
|
||||
|
||||
def __init__(self, params):
|
||||
Expression.__init__(self, params)
|
||||
self.value, self.start, self.length = params
|
||||
|
||||
def __data__(self):
|
||||
return {
|
||||
"sql.substr": [
|
||||
self.value.__data__(),
|
||||
self.start.__data__(),
|
||||
self.length.__data__(),
|
||||
]
|
||||
}
|
||||
|
||||
def vars(self):
|
||||
return self.value.vars() | self.start.vars() | self.length.vars()
|
||||
|
||||
def missing(self):
|
||||
return FALSE
|
||||
Некоторые файлы не были показаны из-за слишком большого количества измененных файлов Показать больше
Загрузка…
Ссылка в новой задаче