From 7a08a42e4e147049b3298b667a6d6a67574dc491 Mon Sep 17 00:00:00 2001 From: Sara Bee <855595+doeg@users.noreply.github.com> Date: Mon, 14 Jun 2021 09:40:04 -0400 Subject: [PATCH 1/4] [vtadmin-web] Add getStreamTablets util Signed-off-by: Sara Bee <855595+doeg@users.noreply.github.com> --- web/vtadmin/src/util/workflows.test.ts | 49 +++++++++++++++++++++++++- web/vtadmin/src/util/workflows.ts | 21 +++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/web/vtadmin/src/util/workflows.test.ts b/web/vtadmin/src/util/workflows.test.ts index 38dc0e3198..3b6014669e 100644 --- a/web/vtadmin/src/util/workflows.test.ts +++ b/web/vtadmin/src/util/workflows.test.ts @@ -14,7 +14,7 @@ * limitations under the License. */ import { vtadmin as pb } from '../proto/vtadmin'; -import { getStreams } from './workflows'; +import { getStreams, getStreamTablets } from './workflows'; describe('getStreams', () => { const tests: { @@ -70,3 +70,50 @@ describe('getStreams', () => { } ); }); + +describe('getStreamTablets', () => { + const tests: { + name: string; + input: Parameters; + expected: ReturnType; + }[] = [ + { + name: 'should return a set of unique tablet aliases', + input: [ + pb.Workflow.create({ + workflow: { + shard_streams: { + '-80/us_east_1a-123456': { + streams: [ + { id: 1, shard: '-80', tablet: { cell: 'us_east_1a', uid: 123456 } }, + { id: 2, shard: '-80', tablet: { cell: 'us_east_1a', uid: 123456 } }, + ], + }, + '80-/us_east_1a-789012': { + streams: [{ id: 1, shard: '80-', tablet: { cell: 'us_east_1a', uid: 789012 } }], + }, + }, + }, + }), + ], + expected: ['us_east_1a-123456', 'us_east_1a-789012'], + }, + { + name: 'should handle empty workflow', + input: [pb.Workflow.create()], + expected: [], + }, + { + name: 'should handle null input', + input: [null], + expected: [], + }, + ]; + + test.each(tests.map(Object.values))( + '%s', + (name: string, input: Parameters, expected: ReturnType) => { + expect(getStreamTablets(...input)).toEqual(expected); + } + ); +}); diff --git a/web/vtadmin/src/util/workflows.ts b/web/vtadmin/src/util/workflows.ts index 61fac25e48..75ed02c693 100644 --- a/web/vtadmin/src/util/workflows.ts +++ b/web/vtadmin/src/util/workflows.ts @@ -59,3 +59,24 @@ export const getTimeUpdated = (workflow: W | null | unde const timestamps = getStreams(workflow).map((s) => parseInt(`${s.time_updated?.seconds}`, 10)); return Math.max(...timestamps); }; + +/** + * getStreamTablets returns an unordered set of tablet alias strings across all streams + * in the workflow. + */ +export const getStreamTablets = (workflow: W | null | undefined): string[] => { + const streams = getStreams(workflow); + if (!Array.isArray(streams)) { + return []; + } + + const aliases = new Set(); + streams.forEach((stream) => { + const alias = formatAlias(stream.tablet); + if (alias) { + aliases.add(alias); + } + }); + + return [...aliases]; +}; From 643174478c18d19def913667de5d24426d4ef45b Mon Sep 17 00:00:00 2001 From: Sara Bee <855595+doeg@users.noreply.github.com> Date: Mon, 14 Jun 2021 10:20:41 -0400 Subject: [PATCH 2/4] [vtadmin-web] Add useManyExperimentalTabletDebugVars query hook Signed-off-by: Sara Bee <855595+doeg@users.noreply.github.com> --- web/vtadmin/src/api/http.ts | 16 ++++++++++--- .../src/components/charts/TabletQPSChart.tsx | 2 +- .../charts/TabletVReplicationQPSChart.tsx | 2 +- web/vtadmin/src/hooks/api.ts | 23 +++++++++++++++---- 4 files changed, 34 insertions(+), 9 deletions(-) diff --git a/web/vtadmin/src/api/http.ts b/web/vtadmin/src/api/http.ts index 61c8c62c0e..d550fb2c6a 100644 --- a/web/vtadmin/src/api/http.ts +++ b/web/vtadmin/src/api/http.ts @@ -18,6 +18,7 @@ import { vtadmin as pb } from '../proto/vtadmin'; import * as errorHandler from '../errors/errorHandler'; import { HttpFetchError, HttpResponseNotOkError, MalformedHttpResponseError } from '../errors/errorTypes'; import { HttpOkResponse } from './responseTypes'; +import { TabletDebugVars } from '../util/tabletDebugVars'; /** * vtfetch makes HTTP requests against the given vtadmin-api endpoint @@ -188,13 +189,22 @@ export const fetchTablet = async ({ clusterID, alias }: FetchTabletParams) => { return pb.Tablet.create(result); }; -export const fetchExperimentalTabletDebugVars = async ({ clusterID, alias }: FetchTabletParams) => { +export interface TabletDebugVarsResponse { + params: FetchTabletParams; + data?: TabletDebugVars; +} + +export const fetchExperimentalTabletDebugVars = async (params: FetchTabletParams): Promise => { if (!process.env.REACT_APP_ENABLE_EXPERIMENTAL_TABLET_DEBUG_VARS) { - return Promise.resolve({}); + return Promise.resolve({ params }); } + const { clusterID, alias } = params; const { result } = await vtfetch(`/api/experimental/tablet/${alias}/debug/vars?cluster=${clusterID}`); - return result; + + // /debug/vars doesn't contain cluster/tablet information, so we + // return that as part of the response. + return { params, data: result }; }; export const fetchTablets = async () => diff --git a/web/vtadmin/src/components/charts/TabletQPSChart.tsx b/web/vtadmin/src/components/charts/TabletQPSChart.tsx index 031a8a15f7..154e2f3260 100644 --- a/web/vtadmin/src/components/charts/TabletQPSChart.tsx +++ b/web/vtadmin/src/components/charts/TabletQPSChart.tsx @@ -37,7 +37,7 @@ export const TabletQPSChart = ({ alias, clusterID }: Props) => { ); const options = useMemo(() => { - const tsdata = getQPSTimeseries(debugVars, query.dataUpdatedAt); + const tsdata = getQPSTimeseries(debugVars?.data, query.dataUpdatedAt); const series: Highcharts.SeriesOptionsType[] = Object.entries(tsdata).map(([name, data]) => ({ data, diff --git a/web/vtadmin/src/components/charts/TabletVReplicationQPSChart.tsx b/web/vtadmin/src/components/charts/TabletVReplicationQPSChart.tsx index 8075e9ba24..8f11818e56 100644 --- a/web/vtadmin/src/components/charts/TabletVReplicationQPSChart.tsx +++ b/web/vtadmin/src/components/charts/TabletVReplicationQPSChart.tsx @@ -37,7 +37,7 @@ export const TabletVReplicationQPSChart = ({ alias, clusterID }: Props) => { ); const options = useMemo(() => { - const tsdata = getVReplicationQPSTimeseries(debugVars, query.dataUpdatedAt); + const tsdata = getVReplicationQPSTimeseries(debugVars?.data, query.dataUpdatedAt); const series: Highcharts.SeriesOptionsType[] = Object.entries(tsdata).map(([name, data]) => ({ data, diff --git a/web/vtadmin/src/hooks/api.ts b/web/vtadmin/src/hooks/api.ts index cce601f538..10d47e3bf3 100644 --- a/web/vtadmin/src/hooks/api.ts +++ b/web/vtadmin/src/hooks/api.ts @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { useQuery, useQueryClient, UseQueryOptions } from 'react-query'; +import { useQueries, useQuery, useQueryClient, UseQueryOptions, UseQueryResult } from 'react-query'; import { fetchClusters, fetchExperimentalTabletDebugVars, @@ -23,15 +23,16 @@ import { FetchSchemaParams, fetchSchemas, fetchTablet, + FetchTabletParams, fetchTablets, fetchVSchema, FetchVSchemaParams, fetchVTExplain, fetchWorkflow, fetchWorkflows, + TabletDebugVarsResponse, } from '../api/http'; import { vtadmin as pb } from '../proto/vtadmin'; -import { TabletDebugVars } from '../util/tabletDebugVars'; import { formatAlias } from '../util/tablets'; /** @@ -81,8 +82,8 @@ export const useTablet = (params: Parameters[0], options?: U }; export const useExperimentalTabletDebugVars = ( - params: Parameters[0], - options?: UseQueryOptions + params: FetchTabletParams, + options?: UseQueryOptions ) => { return useQuery( ['experimental/tablet/debug/vars', params], @@ -91,6 +92,20 @@ export const useExperimentalTabletDebugVars = ( ); }; +export const useManyExperimentalTabletDebugVars = ( + params: FetchTabletParams[], + defaultOptions: UseQueryOptions = {} +) => { + // Robust typing for useQueries is still in progress, so we do + // some sneaky type-casting. See https://github.com/tannerlinsley/react-query/issues/1675 + const queries = params.map((p) => ({ + queryKey: ['experimental/tablet/debug/vars', p], + queryFn: () => fetchExperimentalTabletDebugVars(p), + ...(defaultOptions as any), + })); + return useQueries(queries) as UseQueryResult[]; +}; + /** * useWorkflowsResponse is a query hook that fetches all workflows (by cluster) across every cluster. */ From 18e42597d931d992d5612b3778d668275226cc7e Mon Sep 17 00:00:00 2001 From: Sara Bee <855595+doeg@users.noreply.github.com> Date: Mon, 14 Jun 2021 11:12:47 -0400 Subject: [PATCH 3/4] [vtadmin-web] Add WorkflowStreamsLagChart Signed-off-by: Sara Bee <855595+doeg@users.noreply.github.com> --- .../charts/WorkflowStreamsLagChart.test.tsx | 97 + .../charts/WorkflowStreamsLagChart.tsx | 98 + .../WorkflowStreamsLagChart.test.tsx.snap | 2184 +++++++++++++++++ .../routes/workflow/WorkflowStreams.tsx | 7 +- web/vtadmin/src/util/tabletDebugVars.ts | 6 + 5 files changed, 2391 insertions(+), 1 deletion(-) create mode 100644 web/vtadmin/src/components/charts/WorkflowStreamsLagChart.test.tsx create mode 100644 web/vtadmin/src/components/charts/WorkflowStreamsLagChart.tsx create mode 100644 web/vtadmin/src/components/charts/__snapshots__/WorkflowStreamsLagChart.test.tsx.snap diff --git a/web/vtadmin/src/components/charts/WorkflowStreamsLagChart.test.tsx b/web/vtadmin/src/components/charts/WorkflowStreamsLagChart.test.tsx new file mode 100644 index 0000000000..948b3e0b62 --- /dev/null +++ b/web/vtadmin/src/components/charts/WorkflowStreamsLagChart.test.tsx @@ -0,0 +1,97 @@ +/** + * Copyright 2021 The Vitess Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { UseQueryResult } from 'react-query'; +import { TabletDebugVarsResponse } from '../../api/http'; +import { vtadmin as pb } from '../../proto/vtadmin'; +import { formatSeries } from './WorkflowStreamsLagChart'; + +describe('WorkflowStreamsLagChart', () => { + describe('formatSeries', () => { + it('should return series for all streams in the workflow', () => { + const workflow = pb.Workflow.create({ + cluster: { + id: 'zone1', + name: 'zone1', + }, + workflow: { + shard_streams: { + '-80/us_east_1a-123456': { + streams: [ + { id: 1, shard: '-80', tablet: { cell: 'us_east_1a', uid: 123456 } }, + { id: 2, shard: '-80', tablet: { cell: 'us_east_1a', uid: 123456 } }, + ], + }, + '80-/us_east_1a-789012': { + streams: [{ id: 1, shard: '80-', tablet: { cell: 'us_east_1a', uid: 789012 } }], + }, + }, + }, + }); + + const queries: Partial>[] = [ + { + data: { + params: { alias: 'us_east_1a-123456', clusterID: 'zone1' }, + data: { + VReplicationLag: { + All: [3, 3, 3], + '1': [1, 1, 1], + '2': [2, 2, 2], + }, + }, + }, + dataUpdatedAt: 1000000000000, + }, + { + data: { + params: { alias: 'us_east_1a-789012', clusterID: 'zone1' }, + data: { + VReplicationLag: { + All: [], + '1': [1, 1, 1], + // Some other stream running on the tablet that isn't part + // of this workflow. + '2': [2, 2, 2], + }, + }, + }, + dataUpdatedAt: 1000000000000, + }, + ]; + + // A sneaky cast to UseQueryResult since otherwise enumerating the many fields + // UseQueryResult (most of which we don't use) is pointlessly verbose. + const result = formatSeries(workflow, queries as UseQueryResult[]); + + // Use snapshot matching since defining expected values for arrays of 180 data points is... annoying. + expect(result).toMatchSnapshot(); + + // ...but! Add additional validation so that failing tests are easier to debug. + // (And because it can be tempting to not examine snapshot changes in detail...) :) + expect(result.length).toEqual(3); + + expect(result[0].name).toEqual('us_east_1a-123456/1'); + expect(result[1].name).toEqual('us_east_1a-123456/2'); + expect(result[2].name).toEqual('us_east_1a-789012/1'); + }); + + it('should handle empty input', () => { + const result = formatSeries(null, []); + expect(result).toEqual([]); + }); + }); +}); diff --git a/web/vtadmin/src/components/charts/WorkflowStreamsLagChart.tsx b/web/vtadmin/src/components/charts/WorkflowStreamsLagChart.tsx new file mode 100644 index 0000000000..699983527d --- /dev/null +++ b/web/vtadmin/src/components/charts/WorkflowStreamsLagChart.tsx @@ -0,0 +1,98 @@ +/** + * Copyright 2021 The Vitess Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { useMemo } from 'react'; +import { useManyExperimentalTabletDebugVars, useWorkflow } from '../../hooks/api'; +import { vtadmin } from '../../proto/vtadmin'; +import { getStreamVReplicationLagTimeseries, QPS_REFETCH_INTERVAL } from '../../util/tabletDebugVars'; +import { formatStreamKey, getStreams, getStreamTablets } from '../../util/workflows'; +import { Timeseries } from './Timeseries'; + +interface Props { + clusterID: string; + keyspace: string; + workflowName: string; +} + +export const WorkflowStreamsLagChart = ({ clusterID, keyspace, workflowName }: Props) => { + const { data: workflow, ...wq } = useWorkflow({ clusterID, keyspace, name: workflowName }); + + const queryParams = useMemo(() => { + const aliases = getStreamTablets(workflow); + return aliases.map((alias) => ({ alias, clusterID })); + }, [clusterID, workflow]); + + const tabletQueries = useManyExperimentalTabletDebugVars(queryParams, { + enabled: !!workflow, + refetchInterval: QPS_REFETCH_INTERVAL, + refetchIntervalInBackground: true, + }); + + const anyLoading = wq.isLoading || tabletQueries.some((q) => q.isLoading); + + const chartOptions: Highcharts.Options = useMemo(() => { + return { + series: formatSeries(workflow, tabletQueries), + yAxis: { + labels: { + format: '{text} s', + }, + }, + }; + }, [tabletQueries, workflow]); + + return ; +}; + +export const formatSeries = ( + workflow: vtadmin.Workflow | null | undefined, + tabletQueries: ReturnType +): Highcharts.SeriesOptionsType[] => { + if (!workflow) { + return []; + } + + // Get streamKeys for streams in this workflow. + const streamKeys = getStreams(workflow).map((s) => formatStreamKey(s)); + + return tabletQueries.reduce((acc, tq) => { + if (!tq.data) { + return acc; + } + + const tabletAlias = tq.data.params.alias; + + const lagData = getStreamVReplicationLagTimeseries(tq.data.data, tq.dataUpdatedAt); + Object.entries(lagData).forEach(([streamID, streamLagData]) => { + // Don't graph aggregate vreplication lag for the tablet, since that + // can include vreplication lag data for streams running on the tablet + // that are not in the current workflow. + if (streamID === 'All') { + return; + } + + // Don't graph series for streams that aren't in this workflow. + const streamKey = `${tabletAlias}/${streamID}`; + if (streamKeys.indexOf(streamKey) < 0) { + return; + } + + acc.push({ data: streamLagData, name: streamKey, type: 'line' }); + }); + + return acc; + }, [] as Highcharts.SeriesOptionsType[]); +}; diff --git a/web/vtadmin/src/components/charts/__snapshots__/WorkflowStreamsLagChart.test.tsx.snap b/web/vtadmin/src/components/charts/__snapshots__/WorkflowStreamsLagChart.test.tsx.snap new file mode 100644 index 0000000000..9b60d5253f --- /dev/null +++ b/web/vtadmin/src/components/charts/__snapshots__/WorkflowStreamsLagChart.test.tsx.snap @@ -0,0 +1,2184 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`WorkflowStreamsLagChart formatSeries should return series for all streams in the workflow 1`] = ` +Array [ + Object { + "data": Array [ + Object { + "x": 999997852000, + "y": 0, + }, + Object { + "x": 999997864000, + "y": 0, + }, + Object { + "x": 999997876000, + "y": 0, + }, + Object { + "x": 999997888000, + "y": 0, + }, + Object { + "x": 999997900000, + "y": 0, + }, + Object { + "x": 999997912000, + "y": 0, + }, + Object { + "x": 999997924000, + "y": 0, + }, + Object { + "x": 999997936000, + "y": 0, + }, + Object { + "x": 999997948000, + "y": 0, + }, + Object { + "x": 999997960000, + "y": 0, + }, + Object { + "x": 999997972000, + "y": 0, + }, + Object { + "x": 999997984000, + "y": 0, + }, + Object { + "x": 999997996000, + "y": 0, + }, + Object { + "x": 999998008000, + "y": 0, + }, + Object { + "x": 999998020000, + "y": 0, + }, + Object { + "x": 999998032000, + "y": 0, + }, + Object { + "x": 999998044000, + "y": 0, + }, + Object { + "x": 999998056000, + "y": 0, + }, + Object { + "x": 999998068000, + "y": 0, + }, + Object { + "x": 999998080000, + "y": 0, + }, + Object { + "x": 999998092000, + "y": 0, + }, + Object { + "x": 999998104000, + "y": 0, + }, + Object { + "x": 999998116000, + "y": 0, + }, + Object { + "x": 999998128000, + "y": 0, + }, + Object { + "x": 999998140000, + "y": 0, + }, + Object { + "x": 999998152000, + "y": 0, + }, + Object { + "x": 999998164000, + "y": 0, + }, + Object { + "x": 999998176000, + "y": 0, + }, + Object { + "x": 999998188000, + "y": 0, + }, + Object { + "x": 999998200000, + "y": 0, + }, + Object { + "x": 999998212000, + "y": 0, + }, + Object { + "x": 999998224000, + "y": 0, + }, + Object { + "x": 999998236000, + "y": 0, + }, + Object { + "x": 999998248000, + "y": 0, + }, + Object { + "x": 999998260000, + "y": 0, + }, + Object { + "x": 999998272000, + "y": 0, + }, + Object { + "x": 999998284000, + "y": 0, + }, + Object { + "x": 999998296000, + "y": 0, + }, + Object { + "x": 999998308000, + "y": 0, + }, + Object { + "x": 999998320000, + "y": 0, + }, + Object { + "x": 999998332000, + "y": 0, + }, + Object { + "x": 999998344000, + "y": 0, + }, + Object { + "x": 999998356000, + "y": 0, + }, + Object { + "x": 999998368000, + "y": 0, + }, + Object { + "x": 999998380000, + "y": 0, + }, + Object { + "x": 999998392000, + "y": 0, + }, + Object { + "x": 999998404000, + "y": 0, + }, + Object { + "x": 999998416000, + "y": 0, + }, + Object { + "x": 999998428000, + "y": 0, + }, + Object { + "x": 999998440000, + "y": 0, + }, + Object { + "x": 999998452000, + "y": 0, + }, + Object { + "x": 999998464000, + "y": 0, + }, + Object { + "x": 999998476000, + "y": 0, + }, + Object { + "x": 999998488000, + "y": 0, + }, + Object { + "x": 999998500000, + "y": 0, + }, + Object { + "x": 999998512000, + "y": 0, + }, + Object { + "x": 999998524000, + "y": 0, + }, + Object { + "x": 999998536000, + "y": 0, + }, + Object { + "x": 999998548000, + "y": 0, + }, + Object { + "x": 999998560000, + "y": 0, + }, + Object { + "x": 999998572000, + "y": 0, + }, + Object { + "x": 999998584000, + "y": 0, + }, + Object { + "x": 999998596000, + "y": 0, + }, + Object { + "x": 999998608000, + "y": 0, + }, + Object { + "x": 999998620000, + "y": 0, + }, + Object { + "x": 999998632000, + "y": 0, + }, + Object { + "x": 999998644000, + "y": 0, + }, + Object { + "x": 999998656000, + "y": 0, + }, + Object { + "x": 999998668000, + "y": 0, + }, + Object { + "x": 999998680000, + "y": 0, + }, + Object { + "x": 999998692000, + "y": 0, + }, + Object { + "x": 999998704000, + "y": 0, + }, + Object { + "x": 999998716000, + "y": 0, + }, + Object { + "x": 999998728000, + "y": 0, + }, + Object { + "x": 999998740000, + "y": 0, + }, + Object { + "x": 999998752000, + "y": 0, + }, + Object { + "x": 999998764000, + "y": 0, + }, + Object { + "x": 999998776000, + "y": 0, + }, + Object { + "x": 999998788000, + "y": 0, + }, + Object { + "x": 999998800000, + "y": 0, + }, + Object { + "x": 999998812000, + "y": 0, + }, + Object { + "x": 999998824000, + "y": 0, + }, + Object { + "x": 999998836000, + "y": 0, + }, + Object { + "x": 999998848000, + "y": 0, + }, + Object { + "x": 999998860000, + "y": 0, + }, + Object { + "x": 999998872000, + "y": 0, + }, + Object { + "x": 999998884000, + "y": 0, + }, + Object { + "x": 999998896000, + "y": 0, + }, + Object { + "x": 999998908000, + "y": 0, + }, + Object { + "x": 999998920000, + "y": 0, + }, + Object { + "x": 999998932000, + "y": 0, + }, + Object { + "x": 999998944000, + "y": 0, + }, + Object { + "x": 999998956000, + "y": 0, + }, + Object { + "x": 999998968000, + "y": 0, + }, + Object { + "x": 999998980000, + "y": 0, + }, + Object { + "x": 999998992000, + "y": 0, + }, + Object { + "x": 999999004000, + "y": 0, + }, + Object { + "x": 999999016000, + "y": 0, + }, + Object { + "x": 999999028000, + "y": 0, + }, + Object { + "x": 999999040000, + "y": 0, + }, + Object { + "x": 999999052000, + "y": 0, + }, + Object { + "x": 999999064000, + "y": 0, + }, + Object { + "x": 999999076000, + "y": 0, + }, + Object { + "x": 999999088000, + "y": 0, + }, + Object { + "x": 999999100000, + "y": 0, + }, + Object { + "x": 999999112000, + "y": 0, + }, + Object { + "x": 999999124000, + "y": 0, + }, + Object { + "x": 999999136000, + "y": 0, + }, + Object { + "x": 999999148000, + "y": 0, + }, + Object { + "x": 999999160000, + "y": 0, + }, + Object { + "x": 999999172000, + "y": 0, + }, + Object { + "x": 999999184000, + "y": 0, + }, + Object { + "x": 999999196000, + "y": 0, + }, + Object { + "x": 999999208000, + "y": 0, + }, + Object { + "x": 999999220000, + "y": 0, + }, + Object { + "x": 999999232000, + "y": 0, + }, + Object { + "x": 999999244000, + "y": 0, + }, + Object { + "x": 999999256000, + "y": 0, + }, + Object { + "x": 999999268000, + "y": 0, + }, + Object { + "x": 999999280000, + "y": 0, + }, + Object { + "x": 999999292000, + "y": 0, + }, + Object { + "x": 999999304000, + "y": 0, + }, + Object { + "x": 999999316000, + "y": 0, + }, + Object { + "x": 999999328000, + "y": 0, + }, + Object { + "x": 999999340000, + "y": 0, + }, + Object { + "x": 999999352000, + "y": 0, + }, + Object { + "x": 999999364000, + "y": 0, + }, + Object { + "x": 999999376000, + "y": 0, + }, + Object { + "x": 999999388000, + "y": 0, + }, + Object { + "x": 999999400000, + "y": 0, + }, + Object { + "x": 999999412000, + "y": 0, + }, + Object { + "x": 999999424000, + "y": 0, + }, + Object { + "x": 999999436000, + "y": 0, + }, + Object { + "x": 999999448000, + "y": 0, + }, + Object { + "x": 999999460000, + "y": 0, + }, + Object { + "x": 999999472000, + "y": 0, + }, + Object { + "x": 999999484000, + "y": 0, + }, + Object { + "x": 999999496000, + "y": 0, + }, + Object { + "x": 999999508000, + "y": 0, + }, + Object { + "x": 999999520000, + "y": 0, + }, + Object { + "x": 999999532000, + "y": 0, + }, + Object { + "x": 999999544000, + "y": 0, + }, + Object { + "x": 999999556000, + "y": 0, + }, + Object { + "x": 999999568000, + "y": 0, + }, + Object { + "x": 999999580000, + "y": 0, + }, + Object { + "x": 999999592000, + "y": 0, + }, + Object { + "x": 999999604000, + "y": 0, + }, + Object { + "x": 999999616000, + "y": 0, + }, + Object { + "x": 999999628000, + "y": 0, + }, + Object { + "x": 999999640000, + "y": 0, + }, + Object { + "x": 999999652000, + "y": 0, + }, + Object { + "x": 999999664000, + "y": 0, + }, + Object { + "x": 999999676000, + "y": 0, + }, + Object { + "x": 999999688000, + "y": 0, + }, + Object { + "x": 999999700000, + "y": 0, + }, + Object { + "x": 999999712000, + "y": 0, + }, + Object { + "x": 999999724000, + "y": 0, + }, + Object { + "x": 999999736000, + "y": 0, + }, + Object { + "x": 999999748000, + "y": 0, + }, + Object { + "x": 999999760000, + "y": 0, + }, + Object { + "x": 999999772000, + "y": 0, + }, + Object { + "x": 999999784000, + "y": 0, + }, + Object { + "x": 999999796000, + "y": 0, + }, + Object { + "x": 999999808000, + "y": 0, + }, + Object { + "x": 999999820000, + "y": 0, + }, + Object { + "x": 999999832000, + "y": 0, + }, + Object { + "x": 999999844000, + "y": 0, + }, + Object { + "x": 999999856000, + "y": 0, + }, + Object { + "x": 999999868000, + "y": 0, + }, + Object { + "x": 999999880000, + "y": 0, + }, + Object { + "x": 999999892000, + "y": 0, + }, + Object { + "x": 999999904000, + "y": 0, + }, + Object { + "x": 999999916000, + "y": 0, + }, + Object { + "x": 999999928000, + "y": 0, + }, + Object { + "x": 999999940000, + "y": 0, + }, + Object { + "x": 999999952000, + "y": 0, + }, + Object { + "x": 999999964000, + "y": 0, + }, + Object { + "x": 999999976000, + "y": 1, + }, + Object { + "x": 999999988000, + "y": 1, + }, + Object { + "x": 1000000000000, + "y": 1, + }, + ], + "name": "us_east_1a-123456/1", + "type": "line", + }, + Object { + "data": Array [ + Object { + "x": 999997852000, + "y": 0, + }, + Object { + "x": 999997864000, + "y": 0, + }, + Object { + "x": 999997876000, + "y": 0, + }, + Object { + "x": 999997888000, + "y": 0, + }, + Object { + "x": 999997900000, + "y": 0, + }, + Object { + "x": 999997912000, + "y": 0, + }, + Object { + "x": 999997924000, + "y": 0, + }, + Object { + "x": 999997936000, + "y": 0, + }, + Object { + "x": 999997948000, + "y": 0, + }, + Object { + "x": 999997960000, + "y": 0, + }, + Object { + "x": 999997972000, + "y": 0, + }, + Object { + "x": 999997984000, + "y": 0, + }, + Object { + "x": 999997996000, + "y": 0, + }, + Object { + "x": 999998008000, + "y": 0, + }, + Object { + "x": 999998020000, + "y": 0, + }, + Object { + "x": 999998032000, + "y": 0, + }, + Object { + "x": 999998044000, + "y": 0, + }, + Object { + "x": 999998056000, + "y": 0, + }, + Object { + "x": 999998068000, + "y": 0, + }, + Object { + "x": 999998080000, + "y": 0, + }, + Object { + "x": 999998092000, + "y": 0, + }, + Object { + "x": 999998104000, + "y": 0, + }, + Object { + "x": 999998116000, + "y": 0, + }, + Object { + "x": 999998128000, + "y": 0, + }, + Object { + "x": 999998140000, + "y": 0, + }, + Object { + "x": 999998152000, + "y": 0, + }, + Object { + "x": 999998164000, + "y": 0, + }, + Object { + "x": 999998176000, + "y": 0, + }, + Object { + "x": 999998188000, + "y": 0, + }, + Object { + "x": 999998200000, + "y": 0, + }, + Object { + "x": 999998212000, + "y": 0, + }, + Object { + "x": 999998224000, + "y": 0, + }, + Object { + "x": 999998236000, + "y": 0, + }, + Object { + "x": 999998248000, + "y": 0, + }, + Object { + "x": 999998260000, + "y": 0, + }, + Object { + "x": 999998272000, + "y": 0, + }, + Object { + "x": 999998284000, + "y": 0, + }, + Object { + "x": 999998296000, + "y": 0, + }, + Object { + "x": 999998308000, + "y": 0, + }, + Object { + "x": 999998320000, + "y": 0, + }, + Object { + "x": 999998332000, + "y": 0, + }, + Object { + "x": 999998344000, + "y": 0, + }, + Object { + "x": 999998356000, + "y": 0, + }, + Object { + "x": 999998368000, + "y": 0, + }, + Object { + "x": 999998380000, + "y": 0, + }, + Object { + "x": 999998392000, + "y": 0, + }, + Object { + "x": 999998404000, + "y": 0, + }, + Object { + "x": 999998416000, + "y": 0, + }, + Object { + "x": 999998428000, + "y": 0, + }, + Object { + "x": 999998440000, + "y": 0, + }, + Object { + "x": 999998452000, + "y": 0, + }, + Object { + "x": 999998464000, + "y": 0, + }, + Object { + "x": 999998476000, + "y": 0, + }, + Object { + "x": 999998488000, + "y": 0, + }, + Object { + "x": 999998500000, + "y": 0, + }, + Object { + "x": 999998512000, + "y": 0, + }, + Object { + "x": 999998524000, + "y": 0, + }, + Object { + "x": 999998536000, + "y": 0, + }, + Object { + "x": 999998548000, + "y": 0, + }, + Object { + "x": 999998560000, + "y": 0, + }, + Object { + "x": 999998572000, + "y": 0, + }, + Object { + "x": 999998584000, + "y": 0, + }, + Object { + "x": 999998596000, + "y": 0, + }, + Object { + "x": 999998608000, + "y": 0, + }, + Object { + "x": 999998620000, + "y": 0, + }, + Object { + "x": 999998632000, + "y": 0, + }, + Object { + "x": 999998644000, + "y": 0, + }, + Object { + "x": 999998656000, + "y": 0, + }, + Object { + "x": 999998668000, + "y": 0, + }, + Object { + "x": 999998680000, + "y": 0, + }, + Object { + "x": 999998692000, + "y": 0, + }, + Object { + "x": 999998704000, + "y": 0, + }, + Object { + "x": 999998716000, + "y": 0, + }, + Object { + "x": 999998728000, + "y": 0, + }, + Object { + "x": 999998740000, + "y": 0, + }, + Object { + "x": 999998752000, + "y": 0, + }, + Object { + "x": 999998764000, + "y": 0, + }, + Object { + "x": 999998776000, + "y": 0, + }, + Object { + "x": 999998788000, + "y": 0, + }, + Object { + "x": 999998800000, + "y": 0, + }, + Object { + "x": 999998812000, + "y": 0, + }, + Object { + "x": 999998824000, + "y": 0, + }, + Object { + "x": 999998836000, + "y": 0, + }, + Object { + "x": 999998848000, + "y": 0, + }, + Object { + "x": 999998860000, + "y": 0, + }, + Object { + "x": 999998872000, + "y": 0, + }, + Object { + "x": 999998884000, + "y": 0, + }, + Object { + "x": 999998896000, + "y": 0, + }, + Object { + "x": 999998908000, + "y": 0, + }, + Object { + "x": 999998920000, + "y": 0, + }, + Object { + "x": 999998932000, + "y": 0, + }, + Object { + "x": 999998944000, + "y": 0, + }, + Object { + "x": 999998956000, + "y": 0, + }, + Object { + "x": 999998968000, + "y": 0, + }, + Object { + "x": 999998980000, + "y": 0, + }, + Object { + "x": 999998992000, + "y": 0, + }, + Object { + "x": 999999004000, + "y": 0, + }, + Object { + "x": 999999016000, + "y": 0, + }, + Object { + "x": 999999028000, + "y": 0, + }, + Object { + "x": 999999040000, + "y": 0, + }, + Object { + "x": 999999052000, + "y": 0, + }, + Object { + "x": 999999064000, + "y": 0, + }, + Object { + "x": 999999076000, + "y": 0, + }, + Object { + "x": 999999088000, + "y": 0, + }, + Object { + "x": 999999100000, + "y": 0, + }, + Object { + "x": 999999112000, + "y": 0, + }, + Object { + "x": 999999124000, + "y": 0, + }, + Object { + "x": 999999136000, + "y": 0, + }, + Object { + "x": 999999148000, + "y": 0, + }, + Object { + "x": 999999160000, + "y": 0, + }, + Object { + "x": 999999172000, + "y": 0, + }, + Object { + "x": 999999184000, + "y": 0, + }, + Object { + "x": 999999196000, + "y": 0, + }, + Object { + "x": 999999208000, + "y": 0, + }, + Object { + "x": 999999220000, + "y": 0, + }, + Object { + "x": 999999232000, + "y": 0, + }, + Object { + "x": 999999244000, + "y": 0, + }, + Object { + "x": 999999256000, + "y": 0, + }, + Object { + "x": 999999268000, + "y": 0, + }, + Object { + "x": 999999280000, + "y": 0, + }, + Object { + "x": 999999292000, + "y": 0, + }, + Object { + "x": 999999304000, + "y": 0, + }, + Object { + "x": 999999316000, + "y": 0, + }, + Object { + "x": 999999328000, + "y": 0, + }, + Object { + "x": 999999340000, + "y": 0, + }, + Object { + "x": 999999352000, + "y": 0, + }, + Object { + "x": 999999364000, + "y": 0, + }, + Object { + "x": 999999376000, + "y": 0, + }, + Object { + "x": 999999388000, + "y": 0, + }, + Object { + "x": 999999400000, + "y": 0, + }, + Object { + "x": 999999412000, + "y": 0, + }, + Object { + "x": 999999424000, + "y": 0, + }, + Object { + "x": 999999436000, + "y": 0, + }, + Object { + "x": 999999448000, + "y": 0, + }, + Object { + "x": 999999460000, + "y": 0, + }, + Object { + "x": 999999472000, + "y": 0, + }, + Object { + "x": 999999484000, + "y": 0, + }, + Object { + "x": 999999496000, + "y": 0, + }, + Object { + "x": 999999508000, + "y": 0, + }, + Object { + "x": 999999520000, + "y": 0, + }, + Object { + "x": 999999532000, + "y": 0, + }, + Object { + "x": 999999544000, + "y": 0, + }, + Object { + "x": 999999556000, + "y": 0, + }, + Object { + "x": 999999568000, + "y": 0, + }, + Object { + "x": 999999580000, + "y": 0, + }, + Object { + "x": 999999592000, + "y": 0, + }, + Object { + "x": 999999604000, + "y": 0, + }, + Object { + "x": 999999616000, + "y": 0, + }, + Object { + "x": 999999628000, + "y": 0, + }, + Object { + "x": 999999640000, + "y": 0, + }, + Object { + "x": 999999652000, + "y": 0, + }, + Object { + "x": 999999664000, + "y": 0, + }, + Object { + "x": 999999676000, + "y": 0, + }, + Object { + "x": 999999688000, + "y": 0, + }, + Object { + "x": 999999700000, + "y": 0, + }, + Object { + "x": 999999712000, + "y": 0, + }, + Object { + "x": 999999724000, + "y": 0, + }, + Object { + "x": 999999736000, + "y": 0, + }, + Object { + "x": 999999748000, + "y": 0, + }, + Object { + "x": 999999760000, + "y": 0, + }, + Object { + "x": 999999772000, + "y": 0, + }, + Object { + "x": 999999784000, + "y": 0, + }, + Object { + "x": 999999796000, + "y": 0, + }, + Object { + "x": 999999808000, + "y": 0, + }, + Object { + "x": 999999820000, + "y": 0, + }, + Object { + "x": 999999832000, + "y": 0, + }, + Object { + "x": 999999844000, + "y": 0, + }, + Object { + "x": 999999856000, + "y": 0, + }, + Object { + "x": 999999868000, + "y": 0, + }, + Object { + "x": 999999880000, + "y": 0, + }, + Object { + "x": 999999892000, + "y": 0, + }, + Object { + "x": 999999904000, + "y": 0, + }, + Object { + "x": 999999916000, + "y": 0, + }, + Object { + "x": 999999928000, + "y": 0, + }, + Object { + "x": 999999940000, + "y": 0, + }, + Object { + "x": 999999952000, + "y": 0, + }, + Object { + "x": 999999964000, + "y": 0, + }, + Object { + "x": 999999976000, + "y": 2, + }, + Object { + "x": 999999988000, + "y": 2, + }, + Object { + "x": 1000000000000, + "y": 2, + }, + ], + "name": "us_east_1a-123456/2", + "type": "line", + }, + Object { + "data": Array [ + Object { + "x": 999997852000, + "y": 0, + }, + Object { + "x": 999997864000, + "y": 0, + }, + Object { + "x": 999997876000, + "y": 0, + }, + Object { + "x": 999997888000, + "y": 0, + }, + Object { + "x": 999997900000, + "y": 0, + }, + Object { + "x": 999997912000, + "y": 0, + }, + Object { + "x": 999997924000, + "y": 0, + }, + Object { + "x": 999997936000, + "y": 0, + }, + Object { + "x": 999997948000, + "y": 0, + }, + Object { + "x": 999997960000, + "y": 0, + }, + Object { + "x": 999997972000, + "y": 0, + }, + Object { + "x": 999997984000, + "y": 0, + }, + Object { + "x": 999997996000, + "y": 0, + }, + Object { + "x": 999998008000, + "y": 0, + }, + Object { + "x": 999998020000, + "y": 0, + }, + Object { + "x": 999998032000, + "y": 0, + }, + Object { + "x": 999998044000, + "y": 0, + }, + Object { + "x": 999998056000, + "y": 0, + }, + Object { + "x": 999998068000, + "y": 0, + }, + Object { + "x": 999998080000, + "y": 0, + }, + Object { + "x": 999998092000, + "y": 0, + }, + Object { + "x": 999998104000, + "y": 0, + }, + Object { + "x": 999998116000, + "y": 0, + }, + Object { + "x": 999998128000, + "y": 0, + }, + Object { + "x": 999998140000, + "y": 0, + }, + Object { + "x": 999998152000, + "y": 0, + }, + Object { + "x": 999998164000, + "y": 0, + }, + Object { + "x": 999998176000, + "y": 0, + }, + Object { + "x": 999998188000, + "y": 0, + }, + Object { + "x": 999998200000, + "y": 0, + }, + Object { + "x": 999998212000, + "y": 0, + }, + Object { + "x": 999998224000, + "y": 0, + }, + Object { + "x": 999998236000, + "y": 0, + }, + Object { + "x": 999998248000, + "y": 0, + }, + Object { + "x": 999998260000, + "y": 0, + }, + Object { + "x": 999998272000, + "y": 0, + }, + Object { + "x": 999998284000, + "y": 0, + }, + Object { + "x": 999998296000, + "y": 0, + }, + Object { + "x": 999998308000, + "y": 0, + }, + Object { + "x": 999998320000, + "y": 0, + }, + Object { + "x": 999998332000, + "y": 0, + }, + Object { + "x": 999998344000, + "y": 0, + }, + Object { + "x": 999998356000, + "y": 0, + }, + Object { + "x": 999998368000, + "y": 0, + }, + Object { + "x": 999998380000, + "y": 0, + }, + Object { + "x": 999998392000, + "y": 0, + }, + Object { + "x": 999998404000, + "y": 0, + }, + Object { + "x": 999998416000, + "y": 0, + }, + Object { + "x": 999998428000, + "y": 0, + }, + Object { + "x": 999998440000, + "y": 0, + }, + Object { + "x": 999998452000, + "y": 0, + }, + Object { + "x": 999998464000, + "y": 0, + }, + Object { + "x": 999998476000, + "y": 0, + }, + Object { + "x": 999998488000, + "y": 0, + }, + Object { + "x": 999998500000, + "y": 0, + }, + Object { + "x": 999998512000, + "y": 0, + }, + Object { + "x": 999998524000, + "y": 0, + }, + Object { + "x": 999998536000, + "y": 0, + }, + Object { + "x": 999998548000, + "y": 0, + }, + Object { + "x": 999998560000, + "y": 0, + }, + Object { + "x": 999998572000, + "y": 0, + }, + Object { + "x": 999998584000, + "y": 0, + }, + Object { + "x": 999998596000, + "y": 0, + }, + Object { + "x": 999998608000, + "y": 0, + }, + Object { + "x": 999998620000, + "y": 0, + }, + Object { + "x": 999998632000, + "y": 0, + }, + Object { + "x": 999998644000, + "y": 0, + }, + Object { + "x": 999998656000, + "y": 0, + }, + Object { + "x": 999998668000, + "y": 0, + }, + Object { + "x": 999998680000, + "y": 0, + }, + Object { + "x": 999998692000, + "y": 0, + }, + Object { + "x": 999998704000, + "y": 0, + }, + Object { + "x": 999998716000, + "y": 0, + }, + Object { + "x": 999998728000, + "y": 0, + }, + Object { + "x": 999998740000, + "y": 0, + }, + Object { + "x": 999998752000, + "y": 0, + }, + Object { + "x": 999998764000, + "y": 0, + }, + Object { + "x": 999998776000, + "y": 0, + }, + Object { + "x": 999998788000, + "y": 0, + }, + Object { + "x": 999998800000, + "y": 0, + }, + Object { + "x": 999998812000, + "y": 0, + }, + Object { + "x": 999998824000, + "y": 0, + }, + Object { + "x": 999998836000, + "y": 0, + }, + Object { + "x": 999998848000, + "y": 0, + }, + Object { + "x": 999998860000, + "y": 0, + }, + Object { + "x": 999998872000, + "y": 0, + }, + Object { + "x": 999998884000, + "y": 0, + }, + Object { + "x": 999998896000, + "y": 0, + }, + Object { + "x": 999998908000, + "y": 0, + }, + Object { + "x": 999998920000, + "y": 0, + }, + Object { + "x": 999998932000, + "y": 0, + }, + Object { + "x": 999998944000, + "y": 0, + }, + Object { + "x": 999998956000, + "y": 0, + }, + Object { + "x": 999998968000, + "y": 0, + }, + Object { + "x": 999998980000, + "y": 0, + }, + Object { + "x": 999998992000, + "y": 0, + }, + Object { + "x": 999999004000, + "y": 0, + }, + Object { + "x": 999999016000, + "y": 0, + }, + Object { + "x": 999999028000, + "y": 0, + }, + Object { + "x": 999999040000, + "y": 0, + }, + Object { + "x": 999999052000, + "y": 0, + }, + Object { + "x": 999999064000, + "y": 0, + }, + Object { + "x": 999999076000, + "y": 0, + }, + Object { + "x": 999999088000, + "y": 0, + }, + Object { + "x": 999999100000, + "y": 0, + }, + Object { + "x": 999999112000, + "y": 0, + }, + Object { + "x": 999999124000, + "y": 0, + }, + Object { + "x": 999999136000, + "y": 0, + }, + Object { + "x": 999999148000, + "y": 0, + }, + Object { + "x": 999999160000, + "y": 0, + }, + Object { + "x": 999999172000, + "y": 0, + }, + Object { + "x": 999999184000, + "y": 0, + }, + Object { + "x": 999999196000, + "y": 0, + }, + Object { + "x": 999999208000, + "y": 0, + }, + Object { + "x": 999999220000, + "y": 0, + }, + Object { + "x": 999999232000, + "y": 0, + }, + Object { + "x": 999999244000, + "y": 0, + }, + Object { + "x": 999999256000, + "y": 0, + }, + Object { + "x": 999999268000, + "y": 0, + }, + Object { + "x": 999999280000, + "y": 0, + }, + Object { + "x": 999999292000, + "y": 0, + }, + Object { + "x": 999999304000, + "y": 0, + }, + Object { + "x": 999999316000, + "y": 0, + }, + Object { + "x": 999999328000, + "y": 0, + }, + Object { + "x": 999999340000, + "y": 0, + }, + Object { + "x": 999999352000, + "y": 0, + }, + Object { + "x": 999999364000, + "y": 0, + }, + Object { + "x": 999999376000, + "y": 0, + }, + Object { + "x": 999999388000, + "y": 0, + }, + Object { + "x": 999999400000, + "y": 0, + }, + Object { + "x": 999999412000, + "y": 0, + }, + Object { + "x": 999999424000, + "y": 0, + }, + Object { + "x": 999999436000, + "y": 0, + }, + Object { + "x": 999999448000, + "y": 0, + }, + Object { + "x": 999999460000, + "y": 0, + }, + Object { + "x": 999999472000, + "y": 0, + }, + Object { + "x": 999999484000, + "y": 0, + }, + Object { + "x": 999999496000, + "y": 0, + }, + Object { + "x": 999999508000, + "y": 0, + }, + Object { + "x": 999999520000, + "y": 0, + }, + Object { + "x": 999999532000, + "y": 0, + }, + Object { + "x": 999999544000, + "y": 0, + }, + Object { + "x": 999999556000, + "y": 0, + }, + Object { + "x": 999999568000, + "y": 0, + }, + Object { + "x": 999999580000, + "y": 0, + }, + Object { + "x": 999999592000, + "y": 0, + }, + Object { + "x": 999999604000, + "y": 0, + }, + Object { + "x": 999999616000, + "y": 0, + }, + Object { + "x": 999999628000, + "y": 0, + }, + Object { + "x": 999999640000, + "y": 0, + }, + Object { + "x": 999999652000, + "y": 0, + }, + Object { + "x": 999999664000, + "y": 0, + }, + Object { + "x": 999999676000, + "y": 0, + }, + Object { + "x": 999999688000, + "y": 0, + }, + Object { + "x": 999999700000, + "y": 0, + }, + Object { + "x": 999999712000, + "y": 0, + }, + Object { + "x": 999999724000, + "y": 0, + }, + Object { + "x": 999999736000, + "y": 0, + }, + Object { + "x": 999999748000, + "y": 0, + }, + Object { + "x": 999999760000, + "y": 0, + }, + Object { + "x": 999999772000, + "y": 0, + }, + Object { + "x": 999999784000, + "y": 0, + }, + Object { + "x": 999999796000, + "y": 0, + }, + Object { + "x": 999999808000, + "y": 0, + }, + Object { + "x": 999999820000, + "y": 0, + }, + Object { + "x": 999999832000, + "y": 0, + }, + Object { + "x": 999999844000, + "y": 0, + }, + Object { + "x": 999999856000, + "y": 0, + }, + Object { + "x": 999999868000, + "y": 0, + }, + Object { + "x": 999999880000, + "y": 0, + }, + Object { + "x": 999999892000, + "y": 0, + }, + Object { + "x": 999999904000, + "y": 0, + }, + Object { + "x": 999999916000, + "y": 0, + }, + Object { + "x": 999999928000, + "y": 0, + }, + Object { + "x": 999999940000, + "y": 0, + }, + Object { + "x": 999999952000, + "y": 0, + }, + Object { + "x": 999999964000, + "y": 0, + }, + Object { + "x": 999999976000, + "y": 1, + }, + Object { + "x": 999999988000, + "y": 1, + }, + Object { + "x": 1000000000000, + "y": 1, + }, + ], + "name": "us_east_1a-789012/1", + "type": "line", + }, +] +`; diff --git a/web/vtadmin/src/components/routes/workflow/WorkflowStreams.tsx b/web/vtadmin/src/components/routes/workflow/WorkflowStreams.tsx index b9c8a472a7..23437ac07e 100644 --- a/web/vtadmin/src/components/routes/workflow/WorkflowStreams.tsx +++ b/web/vtadmin/src/components/routes/workflow/WorkflowStreams.tsx @@ -29,6 +29,7 @@ import { DataTable } from '../../dataTable/DataTable'; import { KeyspaceLink } from '../../links/KeyspaceLink'; import { TabletLink } from '../../links/TabletLink'; import { StreamStatePip } from '../../pips/StreamStatePip'; +import { WorkflowStreamsLagChart } from '../../charts/WorkflowStreamsLagChart'; interface Props { clusterID: string; @@ -39,7 +40,7 @@ interface Props { const COLUMNS = ['Stream', 'Source', 'Target', 'Tablet']; export const WorkflowStreams = ({ clusterID, keyspace, name }: Props) => { - const { data } = useWorkflow({ clusterID, keyspace, name }, { refetchInterval: 1000 }); + const { data } = useWorkflow({ clusterID, keyspace, name }); const streams = useMemo(() => { const rows = getStreams(data).map((stream) => ({ @@ -107,6 +108,10 @@ export const WorkflowStreams = ({ clusterID, keyspace, name }: Props) => { return (
+

Stream VReplication Lag

+ + +

Streams

{/* TODO(doeg): add a protobuf enum for this (https://github.com/vitessio/vitess/projects/12#card-60190340) */} {['Error', 'Copying', 'Running', 'Stopped'].map((streamState) => { if (!Array.isArray(streamsByState[streamState])) { diff --git a/web/vtadmin/src/util/tabletDebugVars.ts b/web/vtadmin/src/util/tabletDebugVars.ts index aebf108ca9..37ea08e49b 100644 --- a/web/vtadmin/src/util/tabletDebugVars.ts +++ b/web/vtadmin/src/util/tabletDebugVars.ts @@ -37,6 +37,7 @@ export type TabletDebugVars = Partial<{ QPS: { [k: string]: number[] }; // See https://github.com/vitessio/vitess/blob/main/go/vt/vttablet/tabletmanager/vreplication/stats.go + VReplicationLag: { [k: string]: number[] }; VReplicationQPS: { [k: string]: number[] }; }>; @@ -50,6 +51,11 @@ export type TimeseriesMap = { [seriesName: string]: TimeseriesPoint[] }; export const getQPSTimeseries = (d: TabletDebugVars | null | undefined, endAt?: number): TimeseriesMap => formatTimeseriesMap(d?.QPS || {}, endAt); +export const getStreamVReplicationLagTimeseries = ( + d: TabletDebugVars | null | undefined, + endAt?: number +): TimeseriesMap => formatTimeseriesMap(d?.VReplicationLag || {}, endAt); + export const getVReplicationQPSTimeseries = (d: TabletDebugVars | null | undefined, endAt?: number): TimeseriesMap => formatTimeseriesMap(d?.VReplicationQPS || {}, endAt); From cc331b89a19370d956872ca035a58e8af7323148 Mon Sep 17 00:00:00 2001 From: Sara Bee <855595+doeg@users.noreply.github.com> Date: Mon, 14 Jun 2021 11:54:10 -0400 Subject: [PATCH 4/4] [vtadmin-web] Improve WorkflowStreamsLagChart rendering when no data to show Signed-off-by: Sara Bee <855595+doeg@users.noreply.github.com> --- .../charts/WorkflowStreamsLagChart.tsx | 45 +++++++++++++++---- .../routes/workflow/WorkflowStreams.tsx | 8 +++- web/vtadmin/src/hooks/api.ts | 2 + 3 files changed, 44 insertions(+), 11 deletions(-) diff --git a/web/vtadmin/src/components/charts/WorkflowStreamsLagChart.tsx b/web/vtadmin/src/components/charts/WorkflowStreamsLagChart.tsx index 699983527d..2cfb649195 100644 --- a/web/vtadmin/src/components/charts/WorkflowStreamsLagChart.tsx +++ b/web/vtadmin/src/components/charts/WorkflowStreamsLagChart.tsx @@ -15,6 +15,7 @@ */ import { useMemo } from 'react'; + import { useManyExperimentalTabletDebugVars, useWorkflow } from '../../hooks/api'; import { vtadmin } from '../../proto/vtadmin'; import { getStreamVReplicationLagTimeseries, QPS_REFETCH_INTERVAL } from '../../util/tabletDebugVars'; @@ -27,6 +28,10 @@ interface Props { workflowName: string; } +// Default min/max values (in seconds) for the y-axis when there is no data to show. +const DEFAULT_Y_MAX = 5; +const DEFAULT_Y_MIN = 0; + export const WorkflowStreamsLagChart = ({ clusterID, keyspace, workflowName }: Props) => { const { data: workflow, ...wq } = useWorkflow({ clusterID, keyspace, name: workflowName }); @@ -44,12 +49,22 @@ export const WorkflowStreamsLagChart = ({ clusterID, keyspace, workflowName }: P const anyLoading = wq.isLoading || tabletQueries.some((q) => q.isLoading); const chartOptions: Highcharts.Options = useMemo(() => { + const series = formatSeries(workflow, tabletQueries); + const allSeriesEmpty = series.every((s) => !s.data?.length); + return { - series: formatSeries(workflow, tabletQueries), + series, yAxis: { labels: { format: '{text} s', }, + // The desired behaviour is to show axes + grid lines + // even when there is no data to show. Unfortunately, setting + // softMin/softMax (which is more flexible) doesn't work with showEmpty. + // Instead, we must set explicit min/max, but only when all series are empty. + // If at least one series has data, allow min/max to be automatically calculated. + max: allSeriesEmpty ? DEFAULT_Y_MAX : null, + min: allSeriesEmpty ? DEFAULT_Y_MIN : null, }, }; }, [tabletQueries, workflow]); @@ -57,10 +72,11 @@ export const WorkflowStreamsLagChart = ({ clusterID, keyspace, workflowName }: P return ; }; +// Internal function, exported only for testing. export const formatSeries = ( workflow: vtadmin.Workflow | null | undefined, tabletQueries: ReturnType -): Highcharts.SeriesOptionsType[] => { +): Highcharts.SeriesLineOptions[] => { if (!workflow) { return []; } @@ -68,9 +84,19 @@ export const formatSeries = ( // Get streamKeys for streams in this workflow. const streamKeys = getStreams(workflow).map((s) => formatStreamKey(s)); - return tabletQueries.reduce((acc, tq) => { + // Initialize the timeseries from the workflow, so that every stream in the workflow + // is shown in the legend, even if the /debug/vars data isn't (yet) available. + const seriesByStreamKey: { [streamKey: string]: Highcharts.SeriesLineOptions } = {}; + + streamKeys.forEach((streamKey) => { + if (streamKey) { + seriesByStreamKey[streamKey] = { data: [], name: streamKey, type: 'line' }; + } + }); + + tabletQueries.forEach((tq) => { if (!tq.data) { - return acc; + return; } const tabletAlias = tq.data.params.alias; @@ -84,15 +110,16 @@ export const formatSeries = ( return; } - // Don't graph series for streams that aren't in this workflow. const streamKey = `${tabletAlias}/${streamID}`; - if (streamKeys.indexOf(streamKey) < 0) { + + // Don't graph series for streams that aren't in this workflow. + if (!(streamKey in seriesByStreamKey)) { return; } - acc.push({ data: streamLagData, name: streamKey, type: 'line' }); + seriesByStreamKey[streamKey].data = streamLagData; }); + }); - return acc; - }, [] as Highcharts.SeriesOptionsType[]); + return Object.values(seriesByStreamKey); }; diff --git a/web/vtadmin/src/components/routes/workflow/WorkflowStreams.tsx b/web/vtadmin/src/components/routes/workflow/WorkflowStreams.tsx index 23437ac07e..61cdb5d485 100644 --- a/web/vtadmin/src/components/routes/workflow/WorkflowStreams.tsx +++ b/web/vtadmin/src/components/routes/workflow/WorkflowStreams.tsx @@ -108,8 +108,12 @@ export const WorkflowStreams = ({ clusterID, keyspace, name }: Props) => { return (
-

Stream VReplication Lag

- + {process.env.REACT_APP_ENABLE_EXPERIMENTAL_TABLET_DEBUG_VARS && ( + <> +

Stream VReplication Lag

+ + + )}

Streams

{/* TODO(doeg): add a protobuf enum for this (https://github.com/vitessio/vitess/projects/12#card-60190340) */} diff --git a/web/vtadmin/src/hooks/api.ts b/web/vtadmin/src/hooks/api.ts index 10d47e3bf3..8b2c89f258 100644 --- a/web/vtadmin/src/hooks/api.ts +++ b/web/vtadmin/src/hooks/api.ts @@ -92,6 +92,8 @@ export const useExperimentalTabletDebugVars = ( ); }; +// Future enhancement: add vtadmin-api endpoint to fetch /debug/vars +// for multiple tablets in a single request. https://github.com/vitessio/vitess/projects/12#card-63086674 export const useManyExperimentalTabletDebugVars = ( params: FetchTabletParams[], defaultOptions: UseQueryOptions = {}