updates from ES
This commit is contained in:
Kyle Lahnakoski
Родитель
af5fad48d2
Коммит
34908ed7a8
|
|
@ -27,7 +27,7 @@ path.logs: /data1/logs
|
|||
|
||||
discovery.ec2.endpoint: ec2.us-west-2.amazonaws.com
|
||||
discovery.zen.hosts_provider: ec2
|
||||
discovery.zen.minimum_master_nodes: 1
|
||||
discovery.zen.minimum_master_nodes: 2
|
||||
|
||||
http.compression: true
|
||||
http.cors.enabled: true
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"budget": 3.00, //MAXIMUM SPEND PER HOUR FOR ALL INSTANCES
|
||||
"budget": 4.00, //MAXIMUM SPEND PER HOUR FOR ALL INSTANCES
|
||||
"max_utility_price": 0.02, //MOST THAT WILL BE SPENT ON A SINGLE UTILITY POINT
|
||||
"max_new_utility": 120, //MOST NEW UTILITY THAT WILL BE REQUESTED IN A SINGLE RUN
|
||||
"max_requests_per_type": 2, //LIMIT THE NUMBER OF NET-NEW REQUESTS BY TYPE
|
||||
|
|
@ -19,6 +19,44 @@
|
|||
"more_drives":[
|
||||
{"path":"/data1", "size":1000, "volume_type":"standard"}
|
||||
],
|
||||
"1_nvm_drives":[
|
||||
{"path":"/data1", "device":"/dev/nvme0n1"}
|
||||
],
|
||||
"2_nvm_drives":[
|
||||
{"path":"/data1", "device":"/dev/nvme0n1"},
|
||||
{"path":"/data2", "device":"/dev/nvme1n1"}
|
||||
],
|
||||
"3_nvm_drives":[
|
||||
{"path":"/data1", "device":"/dev/nvme0n1"},
|
||||
{"path":"/data2", "device":"/dev/nvme1n1"},
|
||||
{"path":"/data3", "device":"/dev/nvme2n1"}
|
||||
],
|
||||
"4_nvm_drives":[
|
||||
{"path":"/data1", "device":"/dev/nvme0n1"},
|
||||
{"path":"/data2", "device":"/dev/nvme1n1"},
|
||||
{"path":"/data3", "device":"/dev/nvme2n1"},
|
||||
{"path":"/data4", "device":"/dev/nvme3n1"}
|
||||
],
|
||||
"6_nvm_drives":[
|
||||
{"path":"/data1", "device":"/dev/nvme0n1"},
|
||||
{"path":"/data2", "device":"/dev/nvme1n1"},
|
||||
{"path":"/data3", "device":"/dev/nvme2n1"},
|
||||
{"path":"/data4", "device":"/dev/nvme3n1"},
|
||||
{"path":"/data5", "device":"/dev/nvme4n1"},
|
||||
{"path":"/data6", "device":"/dev/nvme5n1"}
|
||||
],
|
||||
"8_nvm_drives":[
|
||||
{"path":"/data1", "device":"/dev/nvme0n1"},
|
||||
{"path":"/data2", "device":"/dev/nvme1n1"},
|
||||
{"path":"/data3", "device":"/dev/nvme2n1"},
|
||||
{"path":"/data4", "device":"/dev/nvme3n1"},
|
||||
{"path":"/data5", "device":"/dev/nvme4n1"},
|
||||
{"path":"/data6", "device":"/dev/nvme5n1"},
|
||||
{"path":"/data7", "device":"/dev/nvme6n1"},
|
||||
{"path":"/data8", "device":"/dev/nvme7n1"}
|
||||
],
|
||||
|
||||
|
||||
"1_ephemeral_drives":[
|
||||
{"path":"/data1", "device":"/dev/sdb"}
|
||||
],
|
||||
|
|
@ -57,59 +95,97 @@
|
|||
],
|
||||
"utility":[
|
||||
// ONE POINT PER 1 GIG OF MEMORY. OR 2 PER 100 GIG OF DRIVESPACE, OR 60 POINTS, WHICHEVER IS LESS
|
||||
// EBS IS WAY TO SLOW FOR ELASTICSEARCH
|
||||
// {"instance_type": "c1.medium", "storage": 350, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 5, "num_drives": 1, "memory": 1.7, "cpu": 2, "utility": 1.7},
|
||||
// {"instance_type": "c1.xlarge", "storage": 1680, "drives": {"$ref": "#4_ephemeral_drives"}, "discount": 0, "ecu": 20, "num_drives": 4, "memory": 7, "cpu": 8, "utility": 7},
|
||||
{"instance_type": "c3.2xlarge", "storage": 160, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 28, "num_drives": 2, "memory": 15, "cpu": 8, "utility": 3.2},
|
||||
{"instance_type": "c3.4xlarge", "storage": 320, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 55, "num_drives": 2, "memory": 30, "cpu": 16, "utility": 6.4},
|
||||
{"instance_type": "c3.8xlarge", "storage": 640, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 108, "num_drives": 2, "memory": 60, "cpu": 32, "utility": 12.8},
|
||||
{"instance_type": "c3.large", "storage": 32, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 7, "num_drives": 2, "memory": 3.75, "cpu": 2, "utility": 0.64},
|
||||
{"instance_type": "c3.xlarge", "storage": 80, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 14, "num_drives": 2, "memory": 7.5, "cpu": 4, "utility": 1.6},
|
||||
{"blacklist":true, "instance_type": "cc2.8xlarge", "storage": 3360, "drives": {"$ref": "#4_ephemeral_drives"}, "discount": 0, "ecu": 88, "num_drives": 4, "memory": 60.5, "cpu": 32, "utility": 60},
|
||||
// {"instance_type": "cg1.4xlarge", "storage": 1680, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 33.5, "num_drives": 2, "memory": 22.5, "cpu": 16, "utility": 22.5},
|
||||
{"instance_type": "cr1.8xlarge", "storage": 240, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 88, "num_drives": 2, "memory": 244, "cpu": 32, "utility": 4.8},
|
||||
{"instance_type": "d2.2xlarge", "storage": 12000, "drives": {"$ref": "#6_ephemeral_drives"}, "discount": 0, "ecu": 28, "num_drives": 6, "memory": 61, "cpu": 8, "utility": 60},
|
||||
{"instance_type": "d2.4xlarge", "storage": 24000, "drives": {"$ref": "#8_ephemeral_drives"}, "discount": 0, "ecu": 56, "num_drives": 12, "memory": 122, "cpu": 16, "utility": 60},
|
||||
{"instance_type": "d2.8xlarge", "storage": 48000, "drives": {"$ref": "#8_ephemeral_drives"}, "discount": 0, "ecu": 116, "num_drives": 24, "memory": 244, "cpu": 36, "utility": 60},
|
||||
{"instance_type": "d2.xlarge", "storage": 6000, "drives": {"$ref": "#3_ephemeral_drives"}, "discount": 0, "ecu": 14, "num_drives": 3, "memory": 30.5, "cpu": 4, "utility": 30.5},
|
||||
{"instance_type": "g2.2xlarge", "storage": 60, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 26, "num_drives": 1, "memory": 15, "cpu": 8, "utility": 1.2},
|
||||
{"instance_type": "g2.8xlarge", "storage": 240, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 104, "num_drives": 2, "memory": 60, "cpu": 32, "utility": 4.8},
|
||||
{"instance_type": "hi1.4xlarge", "storage": 2048, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 35, "num_drives": 2, "memory": 60.5, "cpu": 16, "utility": 40.96},
|
||||
// {"instance_type": "hs1.8xlarge", "storage": 48000, "drives": {"$ref": "#8_ephemeral_drives"}, "discount": 0, "ecu": 35, "num_drives": 24, "memory": 117, "cpu": 16, "utility": 60},
|
||||
{"instance_type": "i2.2xlarge", "storage": 1600, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 27, "num_drives": 2, "memory": 61, "cpu": 8, "utility": 32.0},
|
||||
{"instance_type": "i2.4xlarge", "storage": 3200, "drives": {"$ref": "#4_ephemeral_drives"}, "discount": 0, "ecu": 53, "num_drives": 4, "memory": 122, "cpu": 16, "utility": 60},
|
||||
{"instance_type": "i2.8xlarge", "storage": 6400, "drives": {"$ref": "#8_ephemeral_drives"}, "discount": 0, "ecu": 104, "num_drives": 8, "memory": 244, "cpu": 32, "utility": 60},
|
||||
{"instance_type": "i2.xlarge", "storage": 800, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 14, "num_drives": 1, "memory": 30.5, "cpu": 4, "utility": 16.0},
|
||||
// {"instance_type": "i3.16xlarge", "storage": 15200, "drives": {"$ref": "#8_ephemeral_drives"}, "discount": 0, "ecu": 200, "num_drives": 8, "memory": 488, "cpu": 64, "utility": 60},
|
||||
// {"instance_type": "i3.2xlarge", "storage": 1900, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 27, "num_drives": 1, "memory": 61, "cpu": 8, "utility": 38.0},
|
||||
// {"instance_type": "i3.4xlarge", "storage": 3800, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 53, "num_drives": 2, "memory": 122, "cpu": 16, "utility": 60},
|
||||
// {"instance_type": "i3.8xlarge", "storage": 7600, "drives": {"$ref": "#4_ephemeral_drives"}, "discount": 0, "ecu": 99, "num_drives": 4, "memory": 244, "cpu": 32, "utility": 60},
|
||||
// {"instance_type": "i3.large", "storage": 475, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 7, "num_drives": 1, "memory": 15.25, "cpu": 2, "utility": 9.5},
|
||||
// {"instance_type": "i3.xlarge", "storage": 950, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 13, "num_drives": 1, "memory": 30.5, "cpu": 4, "utility": 19.0},
|
||||
// {"instance_type": "m1.large", "storage": 840, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 4, "num_drives": 2, "memory": 7.5, "cpu": 2, "utility": 7.5},
|
||||
// {"instance_type": "m1.medium", "storage": 410, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 2, "num_drives": 1, "memory": 3.75, "cpu": 1, "utility": 3.75},
|
||||
// {"instance_type": "m1.small", "storage": 160, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 1, "num_drives": 1, "memory": 1.7, "cpu": 1, "utility": 1.7},
|
||||
// {"instance_type": "m1.xlarge", "storage": 1680, "drives": {"$ref": "#4_ephemeral_drives"}, "discount": 0, "ecu": 8, "num_drives": 4, "memory": 15, "cpu": 4, "utility": 15},
|
||||
// {"instance_type": "m2.2xlarge", "storage": 850, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 13, "num_drives": 1, "memory": 34.2, "cpu": 4, "utility": 17.0},
|
||||
// {"instance_type": "m2.4xlarge", "storage": 1680, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 26, "num_drives": 2, "memory": 68.4, "cpu": 8, "utility": 33.6},
|
||||
// {"instance_type": "m2.xlarge", "storage": 420, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 6.5, "num_drives": 1, "memory": 17.1, "cpu": 2, "utility": 8.4},
|
||||
{"instance_type": "m3.2xlarge", "storage": 160, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 26, "num_drives": 2, "memory": 30, "cpu": 8, "utility": 3.2},
|
||||
{"instance_type": "m3.large", "storage": 32, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 6.5, "num_drives": 1, "memory": 7.5, "cpu": 2, "utility": 0.64},
|
||||
{"instance_type": "m3.medium", "storage": 4, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 3, "num_drives": 1, "memory": 3.75, "cpu": 1, "utility": 0.08},
|
||||
{"instance_type": "m3.xlarge", "storage": 80, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 13, "num_drives": 2, "memory": 15, "cpu": 4, "utility": 1.6},
|
||||
{"instance_type": "r3.2xlarge", "storage": 160, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 26, "num_drives": 1, "memory": 61, "cpu": 8, "utility": 3.2},
|
||||
{"instance_type": "r3.4xlarge", "storage": 320, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 52, "num_drives": 1, "memory": 122, "cpu": 16, "utility": 6.4},
|
||||
{"instance_type": "r3.8xlarge", "storage": 640, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 104, "num_drives": 2, "memory": 244, "cpu": 32, "utility": 12.8},
|
||||
{"instance_type": "r3.large", "storage": 32, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 6.5, "num_drives": 1, "memory": 15.25, "cpu": 2, "utility": 0.64},
|
||||
{"instance_type": "r3.xlarge", "storage": 80, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 13, "num_drives": 1, "memory": 30.5, "cpu": 4, "utility": 1.6},
|
||||
{"instance_type": "x1.16xlarge", "storage": 1920, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "ecu": 174.5, "num_drives": 1, "memory": 976, "cpu": 64, "utility": 38.4},
|
||||
{"instance_type": "x1.32xlarge", "storage": 3840, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "ecu": 349, "num_drives": 2, "memory": 1952, "cpu": 128, "utility": 60}
|
||||
// EBS IS TOO SLOW FOR ELASTICSEARCH
|
||||
|
||||
// {"instance_type": "c3.large", "storage": 32, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "num_drives": 2, "memory": 3.75, "cpu": 2, "utility": 0.64},
|
||||
{"instance_type": "c3.xlarge", "storage": 80, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "num_drives": 2, "memory": 7.5, "cpu": 4, "utility": 1.6},
|
||||
{"instance_type": "c3.2xlarge", "storage": 160, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "num_drives": 2, "memory": 15, "cpu": 8, "utility": 3.2},
|
||||
{"instance_type": "c3.4xlarge", "storage": 320, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "num_drives": 2, "memory": 30, "cpu": 16, "utility": 6.4},
|
||||
{"instance_type": "c3.8xlarge", "storage": 640, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "num_drives": 2, "memory": 60, "cpu": 32, "utility": 12.8},
|
||||
|
||||
{"instance_type": "cc2.8xlarge", "storage": 3360, "drives": {"$ref": "#4_ephemeral_drives"}, "discount": 0, "num_drives": 4, "memory": 60.5, "cpu": 32, "utility": 60},
|
||||
{"instance_type": "cr1.8xlarge", "storage": 240, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "num_drives": 2, "memory": 244, "cpu": 32, "utility": 4.8},
|
||||
|
||||
{"instance_type": "d2.xlarge", "storage": 6000, "drives": {"$ref": "#3_ephemeral_drives"}, "discount": 0, "num_drives": 3, "memory": 30.5, "cpu": 4, "utility": 30.5},
|
||||
{"instance_type": "d2.2xlarge", "storage": 12000, "drives": {"$ref": "#6_ephemeral_drives"}, "discount": 0, "num_drives": 6, "memory": 61, "cpu": 8, "utility": 60},
|
||||
{"instance_type": "d2.4xlarge", "storage": 24000, "drives": {"$ref": "#8_ephemeral_drives"}, "discount": 0, "num_drives": 12, "memory": 122, "cpu": 16, "utility": 60},
|
||||
{"instance_type": "d2.8xlarge", "storage": 48000, "drives": {"$ref": "#8_ephemeral_drives"}, "discount": 0, "num_drives": 24, "memory": 244, "cpu": 36, "utility": 60},
|
||||
|
||||
{"instance_type": "f1.2xlarge", "storage": 470, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 122, "cpu": 8, "utility": 9.4},
|
||||
{"instance_type": "f1.4xlarge", "storage": 940, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 244, "cpu": 16, "utility": 18.8},
|
||||
{"instance_type": "f1.16xlarge", "storage": 3760, "drives": {"$ref": "#4_ephemeral_drives"}, "discount": 0, "num_drives": 4, "memory": 976, "cpu": 64, "utility": 60},
|
||||
|
||||
{"instance_type": "g2.2xlarge", "storage": 60, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 15, "cpu": 8, "utility": 1.2},
|
||||
{"instance_type": "g2.8xlarge", "storage": 240, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "num_drives": 2, "memory": 60, "cpu": 32, "utility": 4.8},
|
||||
|
||||
{"instance_type": "h1.2xlarge", "storage": 2000, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 32, "cpu": 8, "utility": 32},
|
||||
{"instance_type": "h1.4xlarge", "storage": 4000, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "num_drives": 2, "memory": 64, "cpu": 16, "utility": 60},
|
||||
{"instance_type": "h1.8xlarge", "storage": 8000, "drives": {"$ref": "#4_ephemeral_drives"}, "discount": 0, "num_drives": 4, "memory": 128, "cpu": 32, "utility": 60},
|
||||
{"instance_type": "h1.16xlarge", "storage": 16000, "drives": {"$ref": "#8_ephemeral_drives"}, "discount": 0, "num_drives": 8, "memory": 256, "cpu": 64, "utility": 60},
|
||||
|
||||
{"instance_type": "hs1.8xlarge", "storage": 48000, "drives": {"$ref": "#8_ephemeral_drives"}, "discount": 0, "num_drives": 24, "memory": 117, "cpu": 16, "utility": 60},
|
||||
|
||||
{"instance_type": "i2.xlarge", "storage": 800, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 30.5, "cpu": 4, "utility": 16.0},
|
||||
{"instance_type": "i2.2xlarge", "storage": 1600, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "num_drives": 2, "memory": 61, "cpu": 8, "utility": 32.0},
|
||||
{"instance_type": "i2.4xlarge", "storage": 3200, "drives": {"$ref": "#4_ephemeral_drives"}, "discount": 0, "num_drives": 4, "memory": 122, "cpu": 16, "utility": 60},
|
||||
{"instance_type": "i2.8xlarge", "storage": 6400, "drives": {"$ref": "#8_ephemeral_drives"}, "discount": 0, "num_drives": 8, "memory": 244, "cpu": 32, "utility": 60},
|
||||
|
||||
//PROBLEM CONNECTING EPHEMERAL DRIVES
|
||||
{"instance_type": "i3.large", "storage": 475, "drives": {"$ref": "#1_nvm_drives"}, "discount": 0, "num_drives": 1, "memory": 15.25, "cpu": 2, "utility": 9.5},
|
||||
{"instance_type": "i3.xlarge", "storage": 950, "drives": {"$ref": "#1_nvm_drives"}, "discount": 0, "num_drives": 1, "memory": 30.5, "cpu": 4, "utility": 19.0},
|
||||
{"instance_type": "i3.2xlarge", "storage": 1900, "drives": {"$ref": "#1_nvm_drives"}, "discount": 0, "num_drives": 1, "memory": 61, "cpu": 8, "utility": 38.0},
|
||||
{"instance_type": "i3.4xlarge", "storage": 3800, "drives": {"$ref": "#2_nvm_drives"}, "discount": 0, "num_drives": 2, "memory": 122, "cpu": 16, "utility": 60},
|
||||
{"instance_type": "i3.8xlarge", "storage": 7600, "drives": {"$ref": "#4_nvm_drives"}, "discount": 0, "num_drives": 4, "memory": 244, "cpu": 32, "utility": 60},
|
||||
{"instance_type": "i3.16xlarge", "storage": 15200, "drives": {"$ref": "#8_nvm_drives"}, "discount": 0, "num_drives": 8, "memory": 488, "cpu": 64, "utility": 60},
|
||||
|
||||
// NO CAPACITY
|
||||
{"instance_type": "m5d.large", "storage": 75, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 8, "cpu": 2, "utility": 1.5},
|
||||
{"instance_type": "m5d.xlarge", "storage": 150, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 16, "cpu": 4, "utility": 3},
|
||||
{"instance_type": "m5d.2xlarge", "storage": 300, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 32, "cpu": 8, "utility": 6},
|
||||
{"instance_type": "m5d.4xlarge", "storage": 600, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "num_drives": 2, "memory": 64, "cpu": 16, "utility": 12},
|
||||
{"instance_type": "m5d.12xlarge", "storage": 1800, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "num_drives": 2, "memory": 192, "cpu": 48, "utility": 36},
|
||||
{"instance_type": "m5d.24xlarge", "storage": 3600, "drives": {"$ref": "#4_ephemeral_drives"}, "discount": 0, "num_drives": 4, "memory": 384, "cpu": 96, "utility": 60},
|
||||
|
||||
//NO CAPACTIY
|
||||
{"instance_type": "r5d.large", "storage": 75, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 16, "cpu": 2, "utility": 1.5},
|
||||
{"instance_type": "r5d.xlarge", "storage": 150, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 32, "cpu": 4, "utility": 3},
|
||||
{"instance_type": "r5d.2xlarge", "storage": 300, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 64, "cpu": 8, "utility": 6},
|
||||
{"instance_type": "r5d.4xlarge", "storage": 600, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "num_drives": 2, "memory": 128, "cpu": 16, "utility": 12},
|
||||
{"instance_type": "r5d.12xlarge", "storage": 1800, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "num_drives": 2, "memory": 384, "cpu": 48, "utility": 36},
|
||||
{"instance_type": "r5d.24xlarge", "storage": 3600, "drives": {"$ref": "#4_ephemeral_drives"}, "discount": 0, "num_drives": 4, "memory": 768, "cpu": 96, "utility": 60},
|
||||
|
||||
{"instance_type": "c5d.large", "storage": 50, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 4, "cpu": 2, "utility": 1},
|
||||
{"instance_type": "c5d.xlarge", "storage": 100, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 8, "cpu": 4, "utility": 2},
|
||||
{"instance_type": "c5d.2xlarge", "storage": 200, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 16, "cpu": 8, "utility": 4},
|
||||
{"instance_type": "c5d.4xlarge", "storage": 400, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 32, "cpu": 16, "utility": 8},
|
||||
{"instance_type": "c5d.9xlarge", "storage": 900, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 72, "cpu": 36, "utility": 18},
|
||||
{"instance_type": "c5d.18xlarge", "storage": 1800, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "num_drives": 2, "memory": 144, "cpu": 72, "utility": 36},
|
||||
|
||||
{"instance_type": "x1.16xlarge", "storage": 1920, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 976, "cpu": 64, "utility": 38.4},
|
||||
{"instance_type": "x1.32xlarge", "storage": 3840, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "num_drives": 2, "memory": 1952, "cpu": 128, "utility": 60},
|
||||
|
||||
{"instance_type": "x1e.xlarge", "storage": 120, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 122, "cpu": 4, "utility": 2.4},
|
||||
{"instance_type": "x1e.2xlarge", "storage": 240, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 244, "cpu": 8, "utility": 4.8},
|
||||
{"instance_type": "x1e.4xlarge", "storage": 480, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 488, "cpu": 16, "utility": 9.6},
|
||||
{"instance_type": "x1e.8xlarge", "storage": 960, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 976, "cpu": 32, "utility": 19.2},
|
||||
{"instance_type": "x1e.16xlarge", "storage": 1920, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 1952, "cpu": 64, "utility": 38.4},
|
||||
{"instance_type": "x1e.32xlarge", "storage": 3840, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "num_drives": 2, "memory": 3904, "cpu": 128, "utility": 60},
|
||||
|
||||
{"instance_type": "z1d.large", "storage": 75, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 16, "cpu": 2, "utility": 1.5},
|
||||
{"instance_type": "z1d.xlarge", "storage": 150, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 32, "cpu": 4, "utility": 3},
|
||||
{"instance_type": "z1d.2xlarge", "storage": 300, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 64, "cpu": 8, "utility": 6},
|
||||
{"instance_type": "z1d.3xlarge", "storage": 450, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 96, "cpu": 12, "utility": 9},
|
||||
{"instance_type": "z1d.6xlarge", "storage": 900, "drives": {"$ref": "#1_ephemeral_drives"}, "discount": 0, "num_drives": 1, "memory": 192, "cpu": 24, "utility": 18},
|
||||
{"instance_type": "z1d.12xlarge", "storage": 1800, "drives": {"$ref": "#2_ephemeral_drives"}, "discount": 0, "num_drives": 2, "memory": 384, "cpu": 48, "utility": 36}
|
||||
|
||||
|
||||
|
||||
],
|
||||
"ec2": {
|
||||
"request": {
|
||||
//SEE http://boto.readthedocs.org/en/latest/ref/ec2.html#boto.ec2.connection.EC2Connection.request_spot_instances
|
||||
"price": 0.001,
|
||||
"image_id": "ami-bf4193c7",
|
||||
"image_id": "ami-01bbe152bf19d0289", // "ami-bf4193c7",
|
||||
"count": 1,
|
||||
"type": "one-time",
|
||||
"valid_from": null,
|
||||
|
|
@ -166,6 +242,9 @@
|
|||
"encrypted":false
|
||||
}
|
||||
},
|
||||
"constants":{
|
||||
"mo_http.http.default_headers":{"Referer": "https://github.com/klahnakoski/SpotManager"}
|
||||
},
|
||||
"debug": {
|
||||
"trace": true,
|
||||
"cprofile": {
|
||||
|
|
@ -180,6 +259,12 @@
|
|||
"backupCount": 10,
|
||||
"encoding": "utf8"
|
||||
},
|
||||
{
|
||||
"log_type": "elasticsearch",
|
||||
"host": "http://activedata.allizom.org",
|
||||
"index": "debug-spot-manager",
|
||||
"timeout": 600
|
||||
},
|
||||
{
|
||||
"log_type": "ses",
|
||||
"from_address": "klahnakoski@mozilla.com",
|
||||
|
|
|
|||
|
|
@ -41,13 +41,13 @@
|
|||
{"instance_type": "h1.16xlarge", "num_drives": 24, "memory": 256, "cpu": 64, "utility": 64},
|
||||
|
||||
// {"instance_type": "m1.small", "ecu": 3, "num_drives": 1, "memory": 1.7, "cpu": 1, "utility": 1}, # TOO SMALL
|
||||
{"instance_type": "m1.medium", "ecu": 3, "num_drives": 1, "memory": 3.75, "cpu": 1, "utility": 1},
|
||||
{"instance_type": "m1.large", "ecu": 6.5, "num_drives": 2, "memory": 7.5, "cpu": 2, "utility": 2},
|
||||
{"instance_type": "m1.xlarge", "ecu": 13, "num_drives": 4, "memory": 15, "cpu": 4, "utility": 4},
|
||||
|
||||
{"instance_type": "m2.xlarge", "num_drives": 1, "memory": 17.1, "cpu": 2, "utility": 2},
|
||||
{"instance_type": "m2.2xlarge", "num_drives": 1, "memory": 34, "cpu": 4, "utility": 4},
|
||||
{"instance_type": "m2.4xlarge", "num_drives": 2, "memory": 68, "cpu": 8, "utility": 8},
|
||||
// {"instance_type": "m1.medium", "ecu": 3, "num_drives": 1, "memory": 3.75, "cpu": 1, "utility": 1},
|
||||
// {"instance_type": "m1.large", "ecu": 6.5, "num_drives": 2, "memory": 7.5, "cpu": 2, "utility": 2},
|
||||
// {"instance_type": "m1.xlarge", "ecu": 13, "num_drives": 4, "memory": 15, "cpu": 4, "utility": 4},
|
||||
//
|
||||
// {"instance_type": "m2.xlarge", "num_drives": 1, "memory": 17.1, "cpu": 2, "utility": 2},
|
||||
// {"instance_type": "m2.2xlarge", "num_drives": 1, "memory": 34, "cpu": 4, "utility": 4},
|
||||
// {"instance_type": "m2.4xlarge", "num_drives": 2, "memory": 68, "cpu": 8, "utility": 8},
|
||||
|
||||
{"instance_type": "m3.medium", "num_drives": 1, "memory": 3.75, "cpu": 1, "utility": 1},
|
||||
{"instance_type": "m3.large", "num_drives": 1, "memory": 7.5, "cpu": 2, "utility": 2},
|
||||
|
|
@ -127,7 +127,7 @@
|
|||
}
|
||||
},
|
||||
"constants":{
|
||||
"pyLibrary.env.http.default_headers":{"Referer": "https://github.com/klahnakoski/SpotManager"}
|
||||
"mo_http.http.default_headers":{"Referer": "https://github.com/klahnakoski/SpotManager"}
|
||||
},
|
||||
"debug": {
|
||||
"trace": true,
|
||||
|
|
|
|||
|
|
@ -25,6 +25,6 @@
|
|||
},
|
||||
"utility": {"cpu": 1, "utility":1},
|
||||
"constants":{
|
||||
"pyLibrary.env.http.default_headers.referer":"https://wiki.mozilla.org/Auto-tools/Projects/ActiveData"
|
||||
"mo_http.http.default_headers.referer":"https://wiki.mozilla.org/Auto-tools/Projects/ActiveData"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
380
examples/es6.py
380
examples/es6.py
|
|
@ -10,20 +10,18 @@ from __future__ import division
|
|||
from __future__ import unicode_literals
|
||||
|
||||
from mo_fabric import Connection
|
||||
from mo_files import File
|
||||
from mo_future import text_type
|
||||
from mo_files import File, TempFile
|
||||
from mo_future import text
|
||||
from mo_kwargs import override
|
||||
from mo_logs import Log
|
||||
from mo_logs.strings import expand_template
|
||||
from mo_math import Math
|
||||
import mo_math
|
||||
from spot.instance_manager import InstanceManager
|
||||
|
||||
JRE = "jre-8u131-linux-x64.rpm"
|
||||
LOCAL_JRE = "resources/" + JRE
|
||||
|
||||
PYPY_DIR = "pypy-6.0.0-linux_x86_64-portable"
|
||||
PYPY_BZ2 = "pypy-6.0.0-linux_x86_64-portable.tar.bz2"
|
||||
LOCAL_PYPY = "resources/" + PYPY_BZ2
|
||||
PYPY_DIR = "pypy2.7-v7.3.0-linux64"
|
||||
PYPY_BZ2 = "pypy2.7-v7.3.0-linux64.tar.bz2"
|
||||
RESOURCES = File("resources")
|
||||
|
||||
|
||||
class ES6Spot(InstanceManager):
|
||||
|
|
@ -45,14 +43,18 @@ class ES6Spot(InstanceManager):
|
|||
utility, # THE utility OBJECT FOUND IN CONFIG
|
||||
please_stop
|
||||
):
|
||||
with Connection(host=instance.ip_address, kwargs=self.settings.connect) as conn:
|
||||
gigabytes = Math.floor(utility.memory)
|
||||
Log.note("setup {{instance}}", instance=instance.id)
|
||||
try:
|
||||
with Connection(host=instance.ip_address, kwargs=self.settings.connect) as conn:
|
||||
gigabytes = mo_math.floor(utility.memory)
|
||||
Log.note("setup {{instance}}", instance=instance.id)
|
||||
|
||||
self._install_pypy_indexer(instance, conn)
|
||||
self._install_es(gigabytes, instance, conn)
|
||||
self._install_supervisor(instance, conn)
|
||||
self._start_supervisor(conn)
|
||||
_install_pypy_indexer(instance=instance, conn=conn)
|
||||
_install_es(gigabytes, instance=instance, conn=conn)
|
||||
_install_supervisor(instance=instance, conn=conn)
|
||||
_start_supervisor(conn=conn)
|
||||
Log.alert("Done install of {{host}}", host=instance.ip_address)
|
||||
except Exception as e:
|
||||
Log.error("could not setup ES at {{ip}}", ip=instance.ip_address, cause=e)
|
||||
|
||||
def teardown(
|
||||
self,
|
||||
|
|
@ -62,223 +64,221 @@ class ES6Spot(InstanceManager):
|
|||
with Connection(host=instance.ip_address, kwargs=self.settings.connect) as conn:
|
||||
Log.note("teardown {{instance}}", instance=instance.id)
|
||||
|
||||
# ASK NICELY TO STOP Elasticsearch PROCESS
|
||||
conn.sudo("supervisorctl stop push-to-es:*", warn=True)
|
||||
# ASK NICELY TO STOP Elasticsearch PROCESS
|
||||
conn.sudo("supervisorctl stop es:*", warn=True)
|
||||
|
||||
# ASK NICELY TO STOP "supervisord" PROCESS
|
||||
conn.sudo("ps -ef | grep supervisord | grep -v grep | awk '{print $2}' | xargs kill -SIGINT", warn=True)
|
||||
pid = conn.sudo("ps -ef | grep supervisord | grep -v grep | awk '{print $2}'", warn=True).stdout.strip()
|
||||
Log.note("shutdown supervisor at pid={{pid}}", pid=pid)
|
||||
conn.sudo("kill -SIGINT " + pid, warn=True)
|
||||
|
||||
# WAIT FOR SUPERVISOR SHUTDOWN
|
||||
pid = True
|
||||
while pid:
|
||||
pid = conn.sudo("ps -ef | grep supervisord | grep -v grep | awk '{print $2}'")
|
||||
pid = conn.sudo("ps -ef | grep supervisord | grep -v grep | awk '{print $2}'").stdout.strip()
|
||||
|
||||
def _install_es(gigabytes, es_version="6.5.4", instance=None, conn=None):
|
||||
es_file = 'elasticsearch-' + es_version + '.tar.gz'
|
||||
volumes = instance.markup.drives
|
||||
|
||||
def _install_es(self, gigabytes, es_version="6.2.3", instance=None, conn=None):
|
||||
volumes = instance.markup.drives
|
||||
|
||||
if not conn.exists("/usr/local/elasticsearch/config/elasticsearch.yml"):
|
||||
with conn.cd("/home/ec2-user/"):
|
||||
conn.run("mkdir -p temp")
|
||||
|
||||
if not File(LOCAL_JRE).exists:
|
||||
Log.error("Expecting {{file}} on manager to spread to ES instances", file=LOCAL_JRE)
|
||||
response = conn.run("java -version", warn=True)
|
||||
if "Java(TM) SE Runtime Environment" not in response:
|
||||
with conn.cd("/home/ec2-user/temp"):
|
||||
conn.run('rm -f '+JRE)
|
||||
conn.put(LOCAL_JRE, JRE)
|
||||
conn.sudo("rpm -i "+JRE)
|
||||
conn.sudo("alternatives --install /usr/bin/java java /usr/java/default/bin/java 20000")
|
||||
conn.run("export JAVA_HOME=/usr/java/default")
|
||||
|
||||
with conn.cd("/home/ec2-user/"):
|
||||
conn.run('wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-'+es_version+'.tar.gz')
|
||||
conn.run('tar zxfv elasticsearch-'+es_version+'.tar.gz')
|
||||
conn.sudo("rm -fr /usr/local/elasticsearch", warn=True)
|
||||
conn.sudo('mkdir /usr/local/elasticsearch')
|
||||
conn.sudo('cp -R elasticsearch-'+es_version+'/* /usr/local/elasticsearch/')
|
||||
|
||||
with conn.cd('/usr/local/elasticsearch/'):
|
||||
# BE SURE TO MATCH THE PLUGLIN WITH ES VERSION
|
||||
# https://github.com/elasticsearch/elasticsearch-cloud-aws
|
||||
conn.sudo('sudo bin/elasticsearch-plugin install -b discovery-ec2')
|
||||
|
||||
# REMOVE THESE FILES, WE WILL REPLACE THEM WITH THE CORRECT VERSIONS AT THE END
|
||||
conn.sudo("rm -f /usr/local/elasticsearch/config/elasticsearch.yml")
|
||||
conn.sudo("rm -f /usr/local/elasticsearch/config/jvm.options")
|
||||
conn.sudo("rm -f /usr/local/elasticsearch/config/log4j2.properties")
|
||||
|
||||
self.conn = instance.connection
|
||||
|
||||
# MOUNT AND FORMAT THE VOLUMES (list with `lsblk`)
|
||||
for i, k in enumerate(volumes):
|
||||
if not conn.exists(k.path):
|
||||
conn.sudo('sudo umount '+k.device, warn=True)
|
||||
|
||||
conn.sudo('yes | sudo mkfs -t ext4 '+k.device)
|
||||
|
||||
# ES AND JOURNALLING DO NOT MIX
|
||||
conn.sudo('tune2fs -o journal_data_writeback '+k.device)
|
||||
conn.sudo('tune2fs -O ^has_journal '+k.device)
|
||||
conn.sudo('mkdir '+k.path)
|
||||
conn.sudo('sudo mount '+k.device+' '+k.path)
|
||||
conn.sudo('chown -R ec2-user:ec2-user '+k.path)
|
||||
|
||||
# ADD TO /etc/fstab SO AROUND AFTER REBOOT
|
||||
conn.sudo("sed -i '$ a\\"+k.device+" "+k.path+" ext4 defaults,nofail 0 2' /etc/fstab")
|
||||
|
||||
# TEST IT IS WORKING
|
||||
conn.sudo('mount -a')
|
||||
|
||||
# INCREASE THE FILE HANDLE LIMITS
|
||||
if not conn.exists("/usr/local/elasticsearch/config/elasticsearch.yml"):
|
||||
with conn.cd("/home/ec2-user/"):
|
||||
File("./results/temp/sysctl.conf").delete()
|
||||
conn.get("/etc/sysctl.conf", "./results/temp/sysctl.conf", use_sudo=True)
|
||||
lines = File("./results/temp/sysctl.conf").read()
|
||||
conn.run("mkdir -p temp")
|
||||
|
||||
if not (RESOURCES / JRE).exists:
|
||||
Log.error("Expecting {{file}} on manager to spread to ES instances", file=(RESOURCES / JRE))
|
||||
response = conn.run("java -version", warn=True)
|
||||
if "Java(TM) SE Runtime Environment" not in response:
|
||||
with conn.cd("/home/ec2-user/temp"):
|
||||
conn.run('rm -f '+JRE)
|
||||
conn.put((RESOURCES / JRE), JRE)
|
||||
conn.sudo("rpm -i "+JRE)
|
||||
conn.sudo("alternatives --install /usr/bin/java java /usr/java/default/bin/java 20000")
|
||||
conn.run("export JAVA_HOME=/usr/java/default")
|
||||
|
||||
with conn.cd("/home/ec2-user/"):
|
||||
conn.put(RESOURCES / es_file, es_file)
|
||||
conn.run('tar zxfv ' + es_file)
|
||||
conn.sudo("rm -fr /usr/local/elasticsearch", warn=True)
|
||||
conn.sudo('mkdir /usr/local/elasticsearch')
|
||||
conn.sudo('cp -R elasticsearch-'+es_version+'/* /usr/local/elasticsearch/')
|
||||
|
||||
with conn.cd('/usr/local/elasticsearch/'):
|
||||
# BE SURE TO MATCH THE PLUGLIN WITH ES VERSION
|
||||
# https://github.com/elasticsearch/elasticsearch-cloud-aws
|
||||
conn.sudo('sudo bin/elasticsearch-plugin install -b discovery-ec2')
|
||||
|
||||
# REMOVE THESE FILES, WE WILL REPLACE THEM WITH THE CORRECT VERSIONS AT THE END
|
||||
conn.sudo("rm -f /usr/local/elasticsearch/config/elasticsearch.yml")
|
||||
conn.sudo("rm -f /usr/local/elasticsearch/config/jvm.options")
|
||||
conn.sudo("rm -f /usr/local/elasticsearch/config/log4j2.properties")
|
||||
|
||||
# MOUNT AND FORMAT THE VOLUMES (list with `lsblk`)
|
||||
for i, k in enumerate(volumes):
|
||||
if not conn.exists(k.path):
|
||||
# ENSURE DEVICE IS NOT MOUNTED
|
||||
conn.sudo('sudo umount '+k.device, warn=True)
|
||||
|
||||
# (RE)PARTITION THE LOCAL DEVICE, AND FORMAT
|
||||
conn.sudo("parted " + k.device + " --script \"mklabel gpt mkpart primary ext4 2048s 100%\"")
|
||||
conn.sudo('yes | sudo mkfs -t ext4 '+k.device)
|
||||
|
||||
# ES AND JOURNALLING DO NOT MIX
|
||||
conn.sudo('tune2fs -o journal_data_writeback '+k.device)
|
||||
conn.sudo('tune2fs -O ^has_journal '+k.device)
|
||||
|
||||
# MOUNT IT
|
||||
conn.sudo('mkdir '+k.path)
|
||||
conn.sudo('sudo mount '+k.device+' '+k.path)
|
||||
conn.sudo('chown -R ec2-user:ec2-user '+k.path)
|
||||
|
||||
# ADD TO /etc/fstab SO AROUND AFTER REBOOT
|
||||
conn.sudo("sed -i '$ a\\"+k.device+" "+k.path+" ext4 defaults,nofail 0 2' /etc/fstab")
|
||||
|
||||
# TEST IT IS WORKING
|
||||
conn.sudo('mount -a')
|
||||
|
||||
# INCREASE THE FILE HANDLE LIMITS
|
||||
with conn.cd("/home/ec2-user/"):
|
||||
with TempFile() as temp:
|
||||
conn.get("/etc/sysctl.conf", temp, use_sudo=True)
|
||||
lines = temp.read()
|
||||
if lines.find("fs.file-max = 100000") == -1:
|
||||
lines += "\nfs.file-max = 100000"
|
||||
lines = lines.replace("net.bridge.bridge-nf-call-ip6tables = 0", "")
|
||||
lines = lines.replace("net.bridge.bridge-nf-call-iptables = 0", "")
|
||||
lines = lines.replace("net.bridge.bridge-nf-call-arptables = 0", "")
|
||||
File("./results/temp/sysctl.conf").write(lines)
|
||||
conn.put("./results/temp/sysctl.conf", "/etc/sysctl.conf", use_sudo=True)
|
||||
temp.write(lines)
|
||||
conn.put(temp, "/etc/sysctl.conf", use_sudo=True)
|
||||
|
||||
conn.sudo("sudo sed -i '$ a\\vm.max_map_count = 262144' /etc/sysctl.conf")
|
||||
conn.sudo("sudo sed -i '$ a\\vm.max_map_count = 262144' /etc/sysctl.conf")
|
||||
|
||||
conn.sudo("sysctl -p")
|
||||
conn.sudo("sysctl -p")
|
||||
|
||||
# INCREASE FILE HANDLE PERMISSIONS
|
||||
conn.sudo("sed -i '$ a\\root soft nofile 100000' /etc/security/limits.conf")
|
||||
conn.sudo("sed -i '$ a\\root hard nofile 100000' /etc/security/limits.conf")
|
||||
conn.sudo("sed -i '$ a\\root soft memlock unlimited' /etc/security/limits.conf")
|
||||
conn.sudo("sed -i '$ a\\root hard memlock unlimited' /etc/security/limits.conf")
|
||||
# INCREASE FILE HANDLE PERMISSIONS
|
||||
conn.sudo("sed -i '$ a\\root soft nofile 100000' /etc/security/limits.conf")
|
||||
conn.sudo("sed -i '$ a\\root hard nofile 100000' /etc/security/limits.conf")
|
||||
conn.sudo("sed -i '$ a\\root soft memlock unlimited' /etc/security/limits.conf")
|
||||
conn.sudo("sed -i '$ a\\root hard memlock unlimited' /etc/security/limits.conf")
|
||||
|
||||
conn.sudo("sed -i '$ a\\ec2-user soft nofile 100000' /etc/security/limits.conf")
|
||||
conn.sudo("sed -i '$ a\\ec2-user hard nofile 100000' /etc/security/limits.conf")
|
||||
conn.sudo("sed -i '$ a\\ec2-user soft memlock unlimited' /etc/security/limits.conf")
|
||||
conn.sudo("sed -i '$ a\\ec2-user hard memlock unlimited' /etc/security/limits.conf")
|
||||
conn.sudo("sed -i '$ a\\ec2-user soft nofile 100000' /etc/security/limits.conf")
|
||||
conn.sudo("sed -i '$ a\\ec2-user hard nofile 100000' /etc/security/limits.conf")
|
||||
conn.sudo("sed -i '$ a\\ec2-user soft memlock unlimited' /etc/security/limits.conf")
|
||||
conn.sudo("sed -i '$ a\\ec2-user hard memlock unlimited' /etc/security/limits.conf")
|
||||
|
||||
if not conn.exists("/data1/logs"):
|
||||
conn.run('mkdir /data1/logs')
|
||||
conn.run('mkdir /data1/heapdump')
|
||||
|
||||
if not conn.exists("/data1/logs"):
|
||||
conn.run('mkdir /data1/logs')
|
||||
conn.run('mkdir /data1/heapdump')
|
||||
# COPY CONFIG FILES TO ES DIR
|
||||
if not conn.exists("/usr/local/elasticsearch/config/elasticsearch.yml"):
|
||||
conn.put("./examples/config/es6_log4j2.properties", '/usr/local/elasticsearch/config/log4j2.properties', use_sudo=True)
|
||||
|
||||
# COPY CONFIG FILES TO ES DIR
|
||||
if not conn.exists("/usr/local/elasticsearch/config/elasticsearch.yml"):
|
||||
conn.put("./examples/config/es6_log4j2.properties", '/usr/local/elasticsearch/config/log4j2.properties', use_sudo=True)
|
||||
jvm = File("./examples/config/es6_jvm.options").read().replace('\r', '')
|
||||
jvm = expand_template(jvm, {"memory": int(gigabytes/2)})
|
||||
with TempFile() as temp:
|
||||
temp.write(jvm)
|
||||
conn.put(temp, '/usr/local/elasticsearch/config/jvm.options', use_sudo=True)
|
||||
|
||||
jvm = File("./examples/config/es6_jvm.options").read().replace('\r', '')
|
||||
jvm = expand_template(jvm, {"memory": int(gigabytes/2)})
|
||||
File("./results/temp/jvm.options").write(jvm)
|
||||
conn.put("./results/temp/jvm.options", '/usr/local/elasticsearch/config/jvm.options', use_sudo=True)
|
||||
yml = File("./examples/config/es6_config.yml").read().replace("\r", "")
|
||||
yml = expand_template(yml, {
|
||||
"id": instance.ip_address,
|
||||
"data_paths": ",".join("/data" + text(i + 1) for i, _ in enumerate(volumes))
|
||||
})
|
||||
with TempFile() as temp:
|
||||
temp.write(yml)
|
||||
conn.put(temp, '/usr/local/elasticsearch/config/elasticsearch.yml', use_sudo=True)
|
||||
|
||||
yml = File("./examples/config/es6_config.yml").read().replace("\r", "")
|
||||
yml = expand_template(yml, {
|
||||
"id": instance.ip_address,
|
||||
"data_paths": ",".join("/data" + text_type(i + 1) for i, _ in enumerate(volumes))
|
||||
})
|
||||
File("./results/temp/elasticsearch.yml").write(yml)
|
||||
conn.put("./results/temp/elasticsearch.yml", '/usr/local/elasticsearch/config/elasticsearch.yml', use_sudo=True)
|
||||
conn.sudo("chown -R ec2-user:ec2-user /usr/local/elasticsearch")
|
||||
|
||||
conn.sudo("chown -R ec2-user:ec2-user /usr/local/elasticsearch")
|
||||
def _install_python_indexer(instance, conn):
|
||||
Log.note("Install Python at {{instance_id}} ({{address}})", instance_id=instance.id, address=instance.ip_address)
|
||||
_install_python(instance, conn)
|
||||
|
||||
def _install_python_indexer(self, instance, conn):
|
||||
Log.note("Install Python at {{instance_id}} ({{address}})", instance_id=instance.id, address=instance.ip_address)
|
||||
self._install_python(instance, conn)
|
||||
if not conn.exists("/home/ec2-user/ActiveData-ETL/"):
|
||||
with conn.cd("/home/ec2-user"):
|
||||
conn.sudo("yum -y install git")
|
||||
conn.run("git clone https://github.com/klahnakoski/ActiveData-ETL.git")
|
||||
|
||||
if not conn.exists("/home/ec2-user/ActiveData-ETL/"):
|
||||
with conn.cd("/home/ec2-user"):
|
||||
conn.sudo("yum -y install git")
|
||||
conn.run("git clone https://github.com/klahnakoski/ActiveData-ETL.git")
|
||||
with conn.cd("/home/ec2-user/ActiveData-ETL/"):
|
||||
conn.run("git checkout push-to-es6")
|
||||
conn.sudo("yum -y install gcc") # REQUIRED FOR psutil
|
||||
conn.run("~/pypy/bin/pypy -m pip install -r requirements.txt")
|
||||
|
||||
with conn.cd("/home/ec2-user/ActiveData-ETL/"):
|
||||
conn.run("git checkout push-to-es6")
|
||||
conn.sudo("yum -y install gcc") # REQUIRED FOR psutil
|
||||
conn.run("~/pypy/bin/pypy -m pip install -r requirements.txt")
|
||||
conn.put("~/private_active_data_etl.json", "/home/ec2-user/private.json")
|
||||
|
||||
conn.put("~/private_active_data_etl.json", "/home/ec2-user/private.json")
|
||||
def _install_python(instance, conn):
|
||||
Log.note("Install Python at {{instance_id}} ({{address}})", instance_id=instance.id, address=instance.ip_address)
|
||||
if conn.exists("/usr/bin/pip"):
|
||||
pip_version = text(conn.sudo("pip --version", warn=True))
|
||||
else:
|
||||
pip_version = ""
|
||||
|
||||
def _install_python(self, instance, conn):
|
||||
Log.note("Install Python at {{instance_id}} ({{address}})", instance_id=instance.id, address=instance.ip_address)
|
||||
if conn.exists("/usr/bin/pip"):
|
||||
pip_version = conn.sudo("pip --version", warn=True)
|
||||
else:
|
||||
pip_version = ""
|
||||
if not pip_version.startswith("pip 18."):
|
||||
conn.sudo("yum -y install python2")
|
||||
conn.sudo("easy_install pip")
|
||||
# conn.sudo("rm -f /usr/bin/pip", warn=True)
|
||||
# conn.sudo("ln -s /usr/local/bin/pip /usr/bin/pip")
|
||||
# conn.sudo("pip install --upgrade pip")
|
||||
|
||||
if not pip_version.startswith("pip 9."):
|
||||
conn.sudo("yum -y install python27")
|
||||
conn.sudo("easy_install pip")
|
||||
conn.sudo("rm -f /usr/bin/pip", warn=True)
|
||||
conn.sudo("ln -s /usr/local/bin/pip /usr/bin/pip")
|
||||
conn.sudo("pip install --upgrade pip")
|
||||
def _install_pypy(instance, conn):
|
||||
Log.note("Install pypy at {{instance_id}} ({{address}})", instance_id=instance.id, address=instance.ip_address)
|
||||
|
||||
def _install_pypy(self, instance, conn):
|
||||
Log.note("Install pypy at {{instance_id}} ({{address}})", instance_id=instance.id, address=instance.ip_address)
|
||||
if not (RESOURCES / PYPY_BZ2).exists:
|
||||
Log.error("Expecting {{file}} on manager to spread to ES instances", file=(RESOURCES / PYPY_BZ2))
|
||||
|
||||
if not File(LOCAL_PYPY).exists:
|
||||
Log.error("Expecting {{file}} on manager to spread to ES instances", file=LOCAL_PYPY)
|
||||
if conn.exists("~/pypy/bin/pip"):
|
||||
return
|
||||
|
||||
if conn.exists("~/pypy/bin/pip"):
|
||||
return
|
||||
with conn.cd("/home/ec2-user/"):
|
||||
conn.put((RESOURCES / PYPY_BZ2), PYPY_BZ2)
|
||||
conn.run('tar jxf ' + PYPY_BZ2)
|
||||
conn.run("mv " + PYPY_DIR + " pypy")
|
||||
|
||||
with conn.cd("/home/ec2-user/"):
|
||||
conn.put(LOCAL_PYPY, PYPY_BZ2)
|
||||
conn.run('tar jxf ' + PYPY_BZ2)
|
||||
conn.run("mv " + PYPY_DIR + " pypy")
|
||||
conn.run("rm -fr /home/ec2-user/temp", warn=True)
|
||||
conn.run("mkdir /home/ec2-user/temp")
|
||||
with conn.cd("/home/ec2-user/temp"):
|
||||
conn.run("wget https://bootstrap.pypa.io/get-pip.py")
|
||||
conn.run("~/pypy/bin/pypy get-pip.py")
|
||||
|
||||
conn.run("rm -fr /home/ec2-user/temp", warn=True)
|
||||
conn.run("mkdir /home/ec2-user/temp")
|
||||
with conn.cd("/home/ec2-user/temp"):
|
||||
conn.run("wget https://bootstrap.pypa.io/get-pip.py")
|
||||
conn.run("~/pypy/bin/pypy get-pip.py")
|
||||
def _install_pypy_indexer(instance, conn):
|
||||
Log.note("Install indexer at {{instance_id}} ({{address}})", instance_id=instance.id, address=instance.ip_address)
|
||||
_install_pypy(instance, conn)
|
||||
|
||||
def _install_pypy_indexer(self, instance, conn):
|
||||
Log.note("Install indexer at {{instance_id}} ({{address}})", instance_id=instance.id, address=instance.ip_address)
|
||||
self._install_pypy(instance, conn)
|
||||
if not conn.exists("/home/ec2-user/ActiveData-ETL/"):
|
||||
with conn.cd("/home/ec2-user"):
|
||||
conn.sudo("yum -y install git")
|
||||
conn.run("git clone https://github.com/klahnakoski/ActiveData-ETL.git")
|
||||
|
||||
if not conn.exists("/home/ec2-user/ActiveData-ETL/"):
|
||||
with conn.cd("/home/ec2-user"):
|
||||
conn.sudo("yum -y install git")
|
||||
conn.run("git clone https://github.com/klahnakoski/ActiveData-ETL.git")
|
||||
with conn.cd("/home/ec2-user/ActiveData-ETL/"):
|
||||
conn.run("git checkout push-to-es6")
|
||||
conn.run("git pull origin push-to-es6")
|
||||
conn.sudo("yum -y install gcc") # REQUIRED FOR psutil
|
||||
conn.run("~/pypy/bin/pip install -r requirements.txt")
|
||||
|
||||
with conn.cd("/home/ec2-user/ActiveData-ETL/"):
|
||||
conn.run("git checkout push-to-es6")
|
||||
conn.sudo("yum -y install gcc") # REQUIRED FOR psutil
|
||||
conn.run("~/pypy/bin/pip install -r requirements.txt")
|
||||
conn.put("~/private_active_data_etl.json", "/home/ec2-user/private.json")
|
||||
|
||||
conn.put("~/private_active_data_etl.json", "/home/ec2-user/private.json")
|
||||
def _install_supervisor(instance, conn):
|
||||
_install_python(instance, conn)
|
||||
conn.sudo("pip install supervisor")
|
||||
|
||||
def _install_supervisor(self, instance, conn):
|
||||
Log.note("Install Supervisor-plus-Cron at {{instance_id}} ({{address}})", instance_id=instance.id, address=instance.ip_address)
|
||||
# REQUIRED FOR Python SSH
|
||||
self._install_lib("libffi-devel", conn)
|
||||
self._install_lib("openssl-devel", conn)
|
||||
self._install_lib('"Development tools"', install="groupinstall", conn=conn)
|
||||
def _install_lib(lib_name, install="install", conn=None):
|
||||
"""
|
||||
:param lib_name:
|
||||
:param install: use 'groupinstall' if you wish
|
||||
:return:
|
||||
"""
|
||||
result = conn.sudo("yum "+install+" -y "+lib_name, warn=True)
|
||||
if result.return_code != 0 and result.find("already installed and latest version") == -1:
|
||||
Log.error("problem with install of {{lib}}", lib=lib_name)
|
||||
|
||||
self._install_python(instance, conn)
|
||||
conn.sudo("pip install pyopenssl")
|
||||
conn.sudo("pip install ndg-httpsclient")
|
||||
conn.sudo("pip install pyasn1")
|
||||
conn.sudo("pip install fabric==1.10.2")
|
||||
conn.sudo("pip install requests")
|
||||
def _start_supervisor(conn):
|
||||
conn.put("./examples/config/es6_supervisor.conf", "/etc/supervisord.conf", use_sudo=True)
|
||||
|
||||
conn.sudo("pip install supervisor-plus-cron")
|
||||
# START DAEMON (OR THROW ERROR IF RUNNING ALREADY)
|
||||
conn.sudo("supervisord -c /etc/supervisord.conf", warn=True)
|
||||
conn.sudo("supervisorctl reread")
|
||||
conn.sudo("supervisorctl update")
|
||||
|
||||
def _install_lib(self, lib_name, install="install", conn=None):
|
||||
"""
|
||||
:param lib_name:
|
||||
:param install: use 'groupinstall' if you wish
|
||||
:return:
|
||||
"""
|
||||
result = conn.sudo("yum "+install+" -y "+lib_name, warn=True)
|
||||
if result.return_code != 0 and result.find("already installed and latest version") == -1:
|
||||
Log.error("problem with install of {{lib}}", lib=lib_name)
|
||||
|
||||
def _start_supervisor(self, conn):
|
||||
conn.put("./examples/config/es6_supervisor.conf", "/etc/supervisord.conf", use_sudo=True)
|
||||
|
||||
# START DAEMON (OR THROW ERROR IF RUNNING ALREADY)
|
||||
conn.sudo("supervisord -c /etc/supervisord.conf", warn=True)
|
||||
conn.sudo("supervisorctl reread")
|
||||
conn.sudo("supervisorctl update")
|
||||
|
|
|
|||
|
|
@ -9,12 +9,12 @@
|
|||
from __future__ import division
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import mo_math
|
||||
from mo_fabric import Connection
|
||||
from mo_files import File
|
||||
from mo_kwargs import override
|
||||
from mo_logs import Log, constants, startup
|
||||
from mo_logs.strings import between
|
||||
from mo_math import Math
|
||||
from mo_times import Date
|
||||
from pyLibrary import aws
|
||||
from spot.instance_manager import InstanceManager
|
||||
|
|
@ -43,8 +43,7 @@ class ETL(InstanceManager):
|
|||
|
||||
if current_utility < pending / 20:
|
||||
# INCREASE
|
||||
# ENSURE THERE IS PLENTY OF WORK BEFORE MACHINE IS DEPLOYED
|
||||
return max(minimum, Math.ceiling(pending / 20))
|
||||
return max(minimum, mo_math.ceiling(pending / 20)) # ENSURE THERE IS PLENTY OF WORK BEFORE MACHINE IS DEPLOYED
|
||||
else:
|
||||
# DECREASE
|
||||
target = max(minimum, min(current_utility, pending*2))
|
||||
|
|
|
|||
|
|
@ -6,40 +6,46 @@
|
|||
#
|
||||
# Author: Kyle Lahnakoski (kyle@lahnakoski.com)
|
||||
#
|
||||
from __future__ import division
|
||||
from __future__ import unicode_literals
|
||||
from __future__ import division, unicode_literals
|
||||
|
||||
from copy import copy
|
||||
|
||||
import boto
|
||||
import boto.ec2
|
||||
import boto.vpc
|
||||
from boto.ec2.blockdevicemapping import BlockDeviceType, BlockDeviceMapping
|
||||
from boto.ec2.networkinterface import NetworkInterfaceSpecification, NetworkInterfaceCollection
|
||||
from boto.ec2.blockdevicemapping import BlockDeviceMapping, BlockDeviceType
|
||||
from boto.ec2.networkinterface import NetworkInterfaceCollection, NetworkInterfaceSpecification
|
||||
from boto.utils import ISO8601
|
||||
|
||||
import mo_math
|
||||
from jx_python import jx
|
||||
from jx_python.containers.list_usingPythonList import ListContainer
|
||||
from mo_collections import UniqueIndex
|
||||
from mo_dots import unwrap, coalesce, wrap, listwrap, FlatList, Data
|
||||
from mo_dots import Data, FlatList, coalesce, listwrap, unwrap, wrap, Null
|
||||
from mo_dots.objects import datawrap
|
||||
from mo_files import File
|
||||
from mo_future import text_type
|
||||
from mo_future import text
|
||||
from mo_http import http
|
||||
from mo_json import value2json
|
||||
from mo_kwargs import override
|
||||
from mo_logs import Log, startup, Except, constants
|
||||
from mo_logs import Except, Log, constants, startup
|
||||
from mo_logs.startup import SingleInstance
|
||||
from mo_math import Math, MAX, SUM
|
||||
from mo_threads import Lock, Thread, Till, Signal
|
||||
from mo_math import MAX, SUM
|
||||
from mo_threads import Lock, Signal, Thread, Till
|
||||
from mo_threads.threads import MAIN_THREAD
|
||||
from mo_times import MINUTE, Date, Duration, Timer, DAY, WEEK, SECOND, HOUR
|
||||
from mo_times import DAY, Date, Duration, HOUR, MINUTE, SECOND, Timer, WEEK
|
||||
from pyLibrary import convert
|
||||
from pyLibrary.meta import cache, new_instance
|
||||
|
||||
_please_import = http
|
||||
|
||||
ENABLE_SIDE_EFFECTS = True
|
||||
ALLOW_SHUTDOWN = False
|
||||
DEBUG_PRICING = True
|
||||
TIME_FROM_RUNNING_TO_LOGIN = 7 * MINUTE
|
||||
ERROR_ON_CALL_TO_SETUP = "Problem with setup()"
|
||||
DELAY_BEFORE_SETUP = 1 * MINUTE # PROBLEM WITH CONNECTING ONLY HAPPENS WITH BIGGER ES MACHINES
|
||||
CAPACITY_NOT_AVAILABLE_RETRY = Duration("day") # SOME MACHINES ARE NOT AVAILABLE
|
||||
|
||||
|
||||
class SpotManager(object):
|
||||
|
|
@ -57,7 +63,9 @@ class SpotManager(object):
|
|||
self.price_locker = Lock()
|
||||
self.prices = None
|
||||
self.price_lookup = None
|
||||
self.done_spot_requests = Signal()
|
||||
self.no_capacity = {}
|
||||
self.no_capacity_file = File(kwargs.price_file).parent / "no capacity.json"
|
||||
self.done_making_new_spot_requests = Signal()
|
||||
self.net_new_locker = Lock()
|
||||
self.net_new_spot_requests = UniqueIndex(("id",)) # SPOT REQUESTS FOR THIS SESSION
|
||||
self.watcher = None
|
||||
|
|
@ -119,12 +127,12 @@ class SpotManager(object):
|
|||
|
||||
if net_new_utility < 0:
|
||||
if self.settings.allowed_overage:
|
||||
net_new_utility = Math.min(net_new_utility + self.settings.allowed_overage * utility_required, 0)
|
||||
net_new_utility = mo_math.min(net_new_utility + self.settings.allowed_overage * utility_required, 0)
|
||||
|
||||
net_new_utility = self.remove_instances(net_new_utility)
|
||||
|
||||
if net_new_utility > 0:
|
||||
net_new_utility = Math.min(net_new_utility, self.settings.max_new_utility)
|
||||
net_new_utility = mo_math.min(net_new_utility, self.settings.max_new_utility)
|
||||
net_new_utility, remaining_budget = self.add_instances(net_new_utility, remaining_budget)
|
||||
|
||||
if net_new_utility > 0:
|
||||
|
|
@ -142,7 +150,7 @@ class SpotManager(object):
|
|||
req.add_tag("Name", self.settings.ec2.instance.name)
|
||||
|
||||
Log.note("All requests for new utility have been made")
|
||||
self.done_spot_requests.go()
|
||||
self.done_making_new_spot_requests.go()
|
||||
|
||||
def add_instances(self, net_new_utility, remaining_budget):
|
||||
prices = self.pricing()
|
||||
|
|
@ -153,18 +161,18 @@ class SpotManager(object):
|
|||
|
||||
if p.current_price == None:
|
||||
Log.note("{{type}} has no current price",
|
||||
type=p.type.instance_type
|
||||
)
|
||||
type=p.type.instance_type
|
||||
)
|
||||
continue
|
||||
|
||||
if self.settings.utility[p.type.instance_type].blacklist or \
|
||||
p.availability_zone in listwrap(self.settings.utility[p.type.instance_type].blacklist_zones):
|
||||
p.availability_zone in listwrap(self.settings.utility[p.type.instance_type].blacklist_zones):
|
||||
Log.note("{{type}} in {{zone}} skipped due to blacklist", type=p.type.instance_type, zone=p.availability_zone)
|
||||
continue
|
||||
|
||||
# DO NOT BID HIGHER THAN WHAT WE ARE WILLING TO PAY
|
||||
max_acceptable_price = p.type.utility * self.settings.max_utility_price + p.type.discount
|
||||
max_bid = Math.min(p.higher_price, max_acceptable_price, remaining_budget)
|
||||
max_bid = mo_math.min(p.higher_price, max_acceptable_price, remaining_budget)
|
||||
min_bid = p.price_80
|
||||
|
||||
if min_bid > max_acceptable_price:
|
||||
|
|
@ -186,15 +194,15 @@ class SpotManager(object):
|
|||
elif min_bid > max_bid:
|
||||
Log.error("not expected")
|
||||
|
||||
naive_number_needed = int(Math.round(float(net_new_utility) / float(p.type.utility), decimal=0))
|
||||
naive_number_needed = int(mo_math.round(float(net_new_utility) / float(p.type.utility), decimal=0))
|
||||
limit_total = None
|
||||
if self.settings.max_percent_per_type < 1:
|
||||
current_count = sum(1 for a in self.active if a.launch_specification.instance_type == p.type.instance_type and a.launch_specification.placement == p.availability_zone)
|
||||
all_count = sum(1 for a in self.active if a.launch_specification.placement == p.availability_zone)
|
||||
all_count = max(all_count, naive_number_needed)
|
||||
limit_total = int(Math.floor((all_count * self.settings.max_percent_per_type - current_count) / (1 - self.settings.max_percent_per_type)))
|
||||
limit_total = int(mo_math.floor((all_count * self.settings.max_percent_per_type - current_count) / (1 - self.settings.max_percent_per_type)))
|
||||
|
||||
num = Math.min(naive_number_needed, limit_total, self.settings.max_requests_per_type)
|
||||
num = mo_math.min(naive_number_needed, limit_total, self.settings.max_requests_per_type)
|
||||
if num < 0:
|
||||
Log.note(
|
||||
"{{type}} is over {{limit|percent}} of instances, no more requested",
|
||||
|
|
@ -203,10 +211,10 @@ class SpotManager(object):
|
|||
)
|
||||
continue
|
||||
elif num == 1:
|
||||
min_bid = Math.min(Math.max(p.current_price * 1.1, min_bid), max_acceptable_price)
|
||||
min_bid = mo_math.min(mo_math.max(p.current_price * 1.1, min_bid), max_acceptable_price)
|
||||
price_interval = 0
|
||||
else:
|
||||
price_interval = Math.min(min_bid / 10, (max_bid - min_bid) / (num - 1))
|
||||
price_interval = mo_math.min(min_bid / 10, (max_bid - min_bid) / (num - 1))
|
||||
|
||||
for i in range(num):
|
||||
bid_per_machine = min_bid + (i * price_interval)
|
||||
|
|
@ -227,6 +235,15 @@ class SpotManager(object):
|
|||
)
|
||||
continue
|
||||
|
||||
last_no_capacity_message = self.no_capacity.get(p.type.instance_type, Null)
|
||||
if last_no_capacity_message > Date.now() - CAPACITY_NOT_AVAILABLE_RETRY:
|
||||
Log.note(
|
||||
"Did not bid on {{type}}: \"No capacity\" last seen at {{last_time|datetime}}",
|
||||
type=p.type.instance_type,
|
||||
last_time=last_no_capacity_message
|
||||
)
|
||||
continue
|
||||
|
||||
try:
|
||||
if self.settings.ec2.request.count == None or self.settings.ec2.request.count != 1:
|
||||
Log.error("Spot Manager can only request machine one-at-a-time")
|
||||
|
|
@ -288,7 +305,7 @@ class SpotManager(object):
|
|||
Log.note("Shutdown {{instances}}", instances=remove_list.id)
|
||||
remove_threads = [
|
||||
Thread.run(
|
||||
"teardown for " + text_type(i.id),
|
||||
"teardown for " + text(i.id),
|
||||
self.instance_manager.teardown,
|
||||
i
|
||||
)
|
||||
|
|
@ -352,23 +369,24 @@ class SpotManager(object):
|
|||
|
||||
# SEND SHUTDOWN TO EACH INSTANCE
|
||||
Log.warning("Shutdown {{instances}} to save money!", instances=remove_list.id)
|
||||
for g, removals in jx.groupby(remove_list, size=20):
|
||||
for i, t in [
|
||||
(i, Thread.run("teardown " + i.id, self.instance_manager.teardown, i, please_stop=False))
|
||||
for i in removals
|
||||
]:
|
||||
try:
|
||||
t.join()
|
||||
except Exception:
|
||||
Log.note("Problem with shutdown of {{id}}", id=i.id)
|
||||
if ALLOW_SHUTDOWN:
|
||||
for g, removals in jx.groupby(remove_list, size=20):
|
||||
for i, t in [
|
||||
(i, Thread.run("teardown " + i.id, self.instance_manager.teardown, i, please_stop=False))
|
||||
for i in removals
|
||||
]:
|
||||
try:
|
||||
t.join()
|
||||
except Exception:
|
||||
Log.note("Problem with shutdown of {{id}}", id=i.id)
|
||||
|
||||
remove_spot_requests.extend(remove_list.spot_instance_request_id)
|
||||
remove_spot_requests.extend(remove_list.spot_instance_request_id)
|
||||
|
||||
# TERMINATE INSTANCES
|
||||
self.ec2_conn.terminate_instances(instance_ids=remove_list.id)
|
||||
# TERMINATE INSTANCES
|
||||
self.ec2_conn.terminate_instances(instance_ids=remove_list.id)
|
||||
|
||||
# TERMINATE SPOT REQUESTS
|
||||
self.ec2_conn.cancel_spot_instance_requests(request_ids=remove_spot_requests)
|
||||
# TERMINATE SPOT REQUESTS
|
||||
self.ec2_conn.cancel_spot_instance_requests(request_ids=remove_spot_requests)
|
||||
return remaining_budget, net_new_utility
|
||||
|
||||
@cache(duration=5 * SECOND)
|
||||
|
|
@ -389,19 +407,54 @@ class SpotManager(object):
|
|||
return wrap(output)
|
||||
|
||||
def _start_life_cycle_watcher(self):
|
||||
failed_locker = Lock()
|
||||
failed_attempts = Data()
|
||||
|
||||
def track_setup(
|
||||
instance_setup_function,
|
||||
request,
|
||||
instance, # THE boto INSTANCE OBJECT FOR THE MACHINE TO SETUP
|
||||
utility, # THE utility OBJECT FOUND IN CONFIG
|
||||
please_stop
|
||||
):
|
||||
try:
|
||||
instance_setup_function(instance, utility, please_stop)
|
||||
instance.add_tag("Name", self.settings.ec2.instance.name + " (running)")
|
||||
with self.net_new_locker:
|
||||
self.net_new_spot_requests.remove(request.id)
|
||||
except Exception as e:
|
||||
e = Except.wrap(e)
|
||||
instance.add_tag("Name", "")
|
||||
with failed_locker:
|
||||
failed_attempts[request.id] += [e]
|
||||
if "Can not setup unknown " in e:
|
||||
Log.warning("Unexpected failure on startup", instance_id=instance.id, cause=e)
|
||||
elif ERROR_ON_CALL_TO_SETUP in e:
|
||||
with failed_locker:
|
||||
causes = failed_attempts[request.id]
|
||||
if len(causes) > 2:
|
||||
Log.warning("Problem with setup() of {{instance_id}}", instance_id=instance.id, cause=causes)
|
||||
else:
|
||||
Log.warning("Unexpected failure on startup", instance_id=instance.id, cause=e)
|
||||
|
||||
def life_cycle_watcher(please_stop):
|
||||
failed_attempts = Data()
|
||||
bad_requests = Data()
|
||||
setup_threads = []
|
||||
last_get = Date.now()
|
||||
|
||||
while not please_stop:
|
||||
spot_requests = self._get_managed_spot_requests()
|
||||
last_get = Date.now()
|
||||
instances = wrap({i.id: i for r in self.ec2_conn.get_all_instances() for i in r.instances})
|
||||
# INSTANCES THAT REQUIRE SETUP
|
||||
time_to_stop_trying = {}
|
||||
please_setup = [
|
||||
(i, r) for i, r in [(instances[r.instance_id], r) for r in spot_requests]
|
||||
if i.id and not i.tags.get("Name") and i._state.name == "running" and Date.now() > Date(i.launch_time) + DELAY_BEFORE_SETUP
|
||||
if i.id
|
||||
and (
|
||||
not i.tags.get("Name") or i.tags.get("Name") == self.settings.ec2.instance.name + " (setup)"
|
||||
)
|
||||
and i._state.name == "running"
|
||||
and Date.now() > Date(i.launch_time) + DELAY_BEFORE_SETUP
|
||||
]
|
||||
|
||||
for i, r in please_setup:
|
||||
|
|
@ -412,7 +465,7 @@ class SpotManager(object):
|
|||
self.ec2_conn.terminate_instances(instance_ids=[i.id])
|
||||
with self.net_new_locker:
|
||||
self.net_new_spot_requests.remove(r.id)
|
||||
Log.warning("Problem with setup of {{instance_id}}. Time is up. Instance TERMINATED!", instance_id=i.id, cause=e)
|
||||
Log.warning("Problem with setup of {{instance_id}}. Time is up. Instance TERMINATED!", instance_id=i.id)
|
||||
continue
|
||||
|
||||
try:
|
||||
|
|
@ -427,57 +480,35 @@ class SpotManager(object):
|
|||
|
||||
i.markup = p
|
||||
i.add_tag("Name", self.settings.ec2.instance.name + " (setup)")
|
||||
setup_threads.append((i, r, Thread.run(
|
||||
"setup for " + text_type(i.id),
|
||||
setup_threads.append(Thread.run(
|
||||
"setup for " + text(i.id),
|
||||
track_setup,
|
||||
self.instance_manager.setup,
|
||||
r,
|
||||
i,
|
||||
p
|
||||
)))
|
||||
))
|
||||
except Exception as e:
|
||||
i.add_tag("Name", "")
|
||||
Log.warning("Unexpected failure on startup", instance_id=i.id, cause=e)
|
||||
|
||||
please_join = [(i, r, t) for i, r, t in setup_threads if t.stopped]
|
||||
if please_join:
|
||||
Log.note("{{num}} threads have stopped", num=len(please_join))
|
||||
for i, r, t in please_join:
|
||||
try:
|
||||
t.join()
|
||||
setup_threads.remove((i, r, t))
|
||||
i.add_tag("Name", self.settings.ec2.instance.name + " (running)")
|
||||
with self.net_new_locker:
|
||||
self.net_new_spot_requests.remove(r.id)
|
||||
except Exception as e:
|
||||
e = Except.wrap(e)
|
||||
setup_threads.remove((i, r, t))
|
||||
i.add_tag("Name", "")
|
||||
failed_attempts[r.id] += [e]
|
||||
if "Can not setup unknown " in e:
|
||||
Log.warning("Unexpected failure on startup", instance_id=i.id, cause=e)
|
||||
elif ERROR_ON_CALL_TO_SETUP in e:
|
||||
if len(failed_attempts[r.id]) > 2:
|
||||
Log.warning("Problem with setup() of {{instance_id}}", instance_id=i.id, cause=failed_attempts[r.id])
|
||||
else:
|
||||
Log.warning("Unexpected failure on startup", instance_id=i.id, cause=e)
|
||||
|
||||
if Date.now() - last_get > 5 * SECOND:
|
||||
# REFRESH STALE
|
||||
spot_requests = self._get_managed_spot_requests()
|
||||
last_get = Date.now()
|
||||
|
||||
pending = wrap([r for r in spot_requests if r.status.code in PENDING_STATUS_CODES])
|
||||
give_up = wrap([r for r in spot_requests if r.status.code in PROBABLY_NOT_FOR_A_WHILE | TERMINATED_STATUS_CODES])
|
||||
give_up = wrap([r for r in spot_requests if (r.status.code in PROBABLY_NOT_FOR_A_WHILE | TERMINATED_STATUS_CODES) and r.id not in bad_requests])
|
||||
ignore = wrap([r for r in spot_requests if r.status.code in MIGHT_HAPPEN]) # MIGHT HAPPEN, BUT NO NEED TO WAIT FOR IT
|
||||
|
||||
if self.done_spot_requests:
|
||||
if self.done_making_new_spot_requests:
|
||||
with self.net_new_locker:
|
||||
expired = Date.now() - self.settings.run_interval + 2 * MINUTE
|
||||
for ii in list(self.net_new_spot_requests):
|
||||
if Date(ii.create_time) < expired:
|
||||
## SOMETIMES REQUESTS NEVER GET INTO THE MAIN LIST OF REQUESTS
|
||||
# SOMETIMES REQUESTS NEVER GET INTO THE MAIN LIST OF REQUESTS
|
||||
self.net_new_spot_requests.remove(ii)
|
||||
for g in give_up:
|
||||
self.net_new_spot_requests.remove(g.id)
|
||||
|
||||
for g in ignore:
|
||||
self.net_new_spot_requests.remove(g.id)
|
||||
pending = UniqueIndex(("id",), data=pending)
|
||||
|
|
@ -487,17 +518,35 @@ class SpotManager(object):
|
|||
self.ec2_conn.cancel_spot_instance_requests(request_ids=give_up.id)
|
||||
Log.note("Cancelled spot requests {{spots}}, {{reasons}}", spots=give_up.id, reasons=give_up.status.code)
|
||||
|
||||
if not pending and not time_to_stop_trying and self.done_spot_requests and not setup_threads:
|
||||
for g in give_up:
|
||||
bad_requests[g.id] += 1
|
||||
if g.id in self.net_new_spot_requests:
|
||||
self.net_new_spot_requests.remove(g.id)
|
||||
if g.status.code == "capacity-not-available":
|
||||
self.no_capacity[g.launch_specification.instance_type] = Date.now()
|
||||
if g.status.code == "bad-parameters":
|
||||
self.no_capacity[g.launch_specification.instance_type] = Date.now()
|
||||
Log.warning("bad parameters while requesting type {{type}}", type=g.launch_specification.instance_type)
|
||||
|
||||
if not pending and self.done_making_new_spot_requests:
|
||||
Log.note("No more pending spot requests")
|
||||
please_stop.go()
|
||||
break
|
||||
elif setup_threads:
|
||||
Log.note("waiting for setup of {{num}} instances", num=len(setup_threads))
|
||||
elif pending:
|
||||
Log.note("waiting for spot requests: {{pending}}", pending=[p.id for p in pending])
|
||||
|
||||
(Till(seconds=10) | please_stop).wait()
|
||||
|
||||
with Timer("Save no capacity to file"):
|
||||
table = [
|
||||
{"instance_type": k, "last_failure": v}
|
||||
for k, v in self.no_capacity.items()
|
||||
]
|
||||
self.no_capacity_file.write(value2json(table, pretty=True))
|
||||
|
||||
# WAIT FOR SETUP TO COMPLETE
|
||||
for t in setup_threads:
|
||||
t.join()
|
||||
|
||||
Log.note("life cycle watcher has stopped")
|
||||
|
||||
# Log.warning("lifecycle watcher is disabled")
|
||||
|
|
@ -518,6 +567,7 @@ class SpotManager(object):
|
|||
|
||||
@override
|
||||
def _request_spot_instances(self, price, availability_zone_group, instance_type, kwargs):
|
||||
kwargs.self = None
|
||||
kwargs.kwargs = None
|
||||
|
||||
# m3 INSTANCES ARE NOT ALLOWED PLACEMENT GROUP
|
||||
|
|
@ -549,7 +599,7 @@ class SpotManager(object):
|
|||
for i in range(num_ephemeral_volumes):
|
||||
letter = convert.ascii2char(98 + i) # START AT "b"
|
||||
kwargs.block_device_map["/dev/sd" + letter] = BlockDeviceType(
|
||||
ephemeral_name='ephemeral' + text_type(i),
|
||||
ephemeral_name='ephemeral' + text(i),
|
||||
delete_on_termination=True
|
||||
)
|
||||
|
||||
|
|
@ -658,6 +708,17 @@ class SpotManager(object):
|
|||
return self.prices
|
||||
|
||||
def _get_spot_prices_from_aws(self):
|
||||
with Timer("Read no capacity file"):
|
||||
try:
|
||||
# FILE IS LIST OF {instance_type, last_failure} OBJECTS
|
||||
content = self.no_capacity_file.read()
|
||||
self.no_capacity = dict(
|
||||
(r.instance_type, r.last_failure)
|
||||
for r in convert.json2value(content, flexible=False, leaves=False)
|
||||
)
|
||||
except Exception as e:
|
||||
self.no_capacity = {}
|
||||
|
||||
with Timer("Read pricing file"):
|
||||
try:
|
||||
content = File(self.settings.price_file).read()
|
||||
|
|
@ -688,9 +749,9 @@ class SpotManager(object):
|
|||
|
||||
if DEBUG_PRICING:
|
||||
Log.note("get pricing for {{instance_type}} starting at {{start_at}}",
|
||||
instance_type=instance_type,
|
||||
start_at=start_at
|
||||
)
|
||||
instance_type=instance_type,
|
||||
start_at=start_at
|
||||
)
|
||||
|
||||
next_token = None
|
||||
while True:
|
||||
|
|
@ -739,7 +800,7 @@ def find_higher(candidates, reference):
|
|||
|
||||
|
||||
TERMINATED_STATUS_CODES = {
|
||||
"marked-for-termination", # AS GOOD AS DEAD
|
||||
"marked-for-termination", # AS GOOD AS DEAD
|
||||
"capacity-oversubscribed",
|
||||
"capacity-not-available",
|
||||
"instance-terminated-capacity-oversubscribed",
|
||||
|
|
@ -762,8 +823,6 @@ PROBABLY_NOT_FOR_A_WHILE = {
|
|||
"placement-group-constraint",
|
||||
"price-too-low"
|
||||
}
|
||||
|
||||
|
||||
RUNNING_STATUS_CODES = {
|
||||
"fulfilled",
|
||||
"request-canceled-and-instance-running"
|
||||
|
|
@ -773,9 +832,9 @@ RUNNING_STATUS_CODES = {
|
|||
def main():
|
||||
try:
|
||||
settings = startup.read_settings()
|
||||
constants.set(settings.constants)
|
||||
Log.start(settings.debug)
|
||||
with SingleInstance(flavor_id=settings.args.filename):
|
||||
constants.set(settings.constants)
|
||||
settings.run_interval = Duration(settings.run_interval)
|
||||
for u in settings.utility:
|
||||
u.discount = coalesce(u.discount, 0)
|
||||
|
|
@ -804,31 +863,35 @@ def main():
|
|||
ephemeral_storage = {
|
||||
"c1.medium": {"num": 1, "size": 350},
|
||||
"c1.xlarge": {"num": 4, "size": 1680},
|
||||
|
||||
"c3.2xlarge": {"num": 2, "size": 160},
|
||||
"c3.4xlarge": {"num": 2, "size": 320},
|
||||
"c3.8xlarge": {"num": 2, "size": 640},
|
||||
"c3.large": {"num": 2, "size": 32},
|
||||
"c3.xlarge": {"num": 2, "size": 80},
|
||||
|
||||
"c4.2xlarge": {"num": 0, "size": 0},
|
||||
"c4.4xlarge": {"num": 0, "size": 0},
|
||||
"c4.8xlarge": {"num": 0, "size": 0},
|
||||
"c4.large": {"num": 0, "size": 0},
|
||||
"c4.xlarge": {"num": 0, "size": 0},
|
||||
|
||||
"c5.large":{"num": 0, "size": 0},
|
||||
"c5.xlarge":{"num": 0, "size": 0},
|
||||
"c5.2xlarge":{"num": 0, "size": 0},
|
||||
"c5.4xlarge":{"num": 0, "size": 0},
|
||||
"c5.9xlarge":{"num": 0, "size": 0},
|
||||
"c5.18xlarge":{"num": 0, "size": 0},
|
||||
|
||||
"cc2.8xlarge": {"num": 4, "size": 3360},
|
||||
"cg1.4xlarge": {"num": 2, "size": 1680},
|
||||
"cr1.8xlarge": {"num": 2, "size": 240},
|
||||
|
||||
"d2.2xlarge": {"num": 6, "size": 12000},
|
||||
"d2.4xlarge": {"num": 12, "size": 24000},
|
||||
"d2.8xlarge": {"num": 24, "size": 48000},
|
||||
"d2.xlarge": {"num": 3, "size": 6000},
|
||||
"f1.16xlarge": {"num": 0, "size": 0},
|
||||
"f1.2xlarge": {"num": 0, "size": 0},
|
||||
|
||||
"g2.2xlarge": {"num": 1, "size": 60},
|
||||
"g2.8xlarge": {"num": 2, "size": 240},
|
||||
"h1.2xlarge": {"num": 1, "size": 2000},
|
||||
|
|
@ -842,19 +905,18 @@ ephemeral_storage = {
|
|||
"i2.4xlarge": {"num": 4, "size": 3200},
|
||||
"i2.8xlarge": {"num": 8, "size": 6400},
|
||||
"i2.xlarge": {"num": 1, "size": 800},
|
||||
"i3.16xlarge": {"num": 8, "size": 15200},
|
||||
|
||||
"i3.large": {"num": 1, "size": 475},
|
||||
"i3.xlarge": {"num": 1, "size": 950},
|
||||
"i3.2xlarge": {"num": 1, "size": 1900},
|
||||
"i3.4xlarge": {"num": 2, "size": 3800},
|
||||
"i3.8xlarge": {"num": 4, "size": 7600},
|
||||
"i3.large": {"num": 1, "size": 475},
|
||||
"i3.xlarge": {"num": 1, "size": 950},
|
||||
"m1.large": {"num": 2, "size": 840},
|
||||
"m1.medium": {"num": 1, "size": 410},
|
||||
"m1.small": {"num": 1, "size": 160},
|
||||
"m1.xlarge": {"num": 4, "size": 1680},
|
||||
"m2.2xlarge": {"num": 1, "size": 850},
|
||||
"m2.4xlarge": {"num": 2, "size": 1680},
|
||||
"m2.xlarge": {"num": 1, "size": 420},
|
||||
"i3.16xlarge": {"num": 8, "size": 15200},
|
||||
|
||||
"f1.2xlarge": {"num": 1, "size": 470},
|
||||
"f1.4xlarge": {"num": 1, "size": 940},
|
||||
"f1.16xlarge": {"num": 4, "size": 940},
|
||||
|
||||
"m3.2xlarge": {"num": 2, "size": 160},
|
||||
"m3.large": {"num": 1, "size": 32},
|
||||
"m3.medium": {"num": 1, "size": 4},
|
||||
|
|
@ -865,6 +927,14 @@ ephemeral_storage = {
|
|||
"m4.4xlarge": {"num": 0, "size": 0},
|
||||
"m4.large": {"num": 0, "size": 0},
|
||||
"m4.xlarge": {"num": 0, "size": 0},
|
||||
|
||||
"m5d.large": {"num": 1, "size": 75},
|
||||
"m5d.xlarge": {"num": 1, "size": 150},
|
||||
"m5d.2xlarge": {"num": 1, "size": 300},
|
||||
"m5d.4xlarge": {"num": 2, "size": 300},
|
||||
"m5d.12xlarge": {"num": 2, "size": 900},
|
||||
"m5d.24xlarge": {"num": 4, "size": 900},
|
||||
|
||||
"p2.16xlarge": {"num": 0, "size": 0},
|
||||
"p2.8xlarge": {"num": 0, "size": 0},
|
||||
"p2.xlarge": {"num": 0, "size": 0},
|
||||
|
|
@ -879,6 +949,14 @@ ephemeral_storage = {
|
|||
"r4.8xlarge": {"num": 0, "size": 0},
|
||||
"r4.large": {"num": 0, "size": 0},
|
||||
"r4.xlarge": {"num": 0, "size": 0},
|
||||
|
||||
"r5d.large": {"num": 1, "size": 75},
|
||||
"r5d.xlarge": {"num": 1, "size": 150},
|
||||
"r5d.2xlarge": {"num": 1, "size": 300},
|
||||
"r5d.4xlarge": {"num": 2, "size": 300},
|
||||
"r5d.12xlarge": {"num": 2, "size": 900},
|
||||
"r5d.24xlarge": {"num": 4, "size": 900},
|
||||
|
||||
"t1.micro": {"num": 0, "size": 0},
|
||||
"t2.2xlarge": {"num": 0, "size": 0},
|
||||
"t2.large": {"num": 0, "size": 0},
|
||||
|
|
@ -887,6 +965,28 @@ ephemeral_storage = {
|
|||
"t2.nano": {"num": 0, "size": 0},
|
||||
"t2.small": {"num": 0, "size": 0},
|
||||
"t2.xlarge": {"num": 0, "size": 0},
|
||||
|
||||
"c5d.large": {"num": 1, "size": 50},
|
||||
"c5d.xlarge": {"num": 1, "size": 100},
|
||||
"c5d.2xlarge": {"num": 1, "size": 200},
|
||||
"c5d.4xlarge": {"num": 1, "size": 400},
|
||||
"c5d.9xlarge": {"num": 1, "size": 900},
|
||||
"c5d.18xlarge": {"num": 2, "size": 900},
|
||||
|
||||
"x1e.xlarge": {"num": 1, "size": 120},
|
||||
"x1e.2xlarge": {"num": 1, "size": 240},
|
||||
"x1e.4xlarge": {"num": 1, "size": 480},
|
||||
"x1e.8xlarge": {"num": 1, "size": 960},
|
||||
"x1e.16xlarge": {"num": 1, "size": 1920},
|
||||
"x1e.32xlarge": {"num": 2, "size": 1920},
|
||||
|
||||
"z1d.large": {"num": 1, "size": 75},
|
||||
"z1d.xlarge": {"num": 1, "size": 150},
|
||||
"z1d.2xlarge": {"num": 1, "size": 300},
|
||||
"z1d.3xlarge": {"num": 1, "size": 450},
|
||||
"z1d.6xlarge": {"num": 1, "size": 900},
|
||||
"z1d.12xlarge": {"num": 1, "size": 900},
|
||||
|
||||
"x1.16xlarge": {"num": 1, "size": 1920},
|
||||
"x1.32xlarge": {"num": 2, "size": 3840}
|
||||
}
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче