microsoft
/
gather
зеркало из https://github.com/microsoft/gather.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

UI: highlight dirty dependencies in different color

This commit is contained in:
Andrew Head 2019-03-04 18:15:13 -08:00
Родитель db3b7b33b3
Коммит fe6c1098af
13 изменённых файлов: 156 добавлений и 81 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

16
src/analysis/slice/log-slicer.ts
Просмотреть файл

@ -1,3 +1,4 @@
import { Signal } from "@phosphor/signaling";
import { ICell } from "../../model/cell";
import { CellSlice } from "../../model/cellslice";
import { DataflowAnalyzer } from "./data-flow";
@ -63,6 +64,11 @@ export class ExecutionLogSlicer {
public _programBuilder: ProgramBuilder;
private _dataflowAnalyzer: DataflowAnalyzer;
/**
* Signal emitted when a cell's execution has been completely processed.
*/
readonly executionLogged = new Signal<this, CellExecution>(this);
/**
* Construct a new execution log slicer.
*/
@ -72,7 +78,8 @@ export class ExecutionLogSlicer {
}
/**
* Log that a cell has just been executed.
* Log that a cell has just been executed. The execution time for this cell will be stored
* as the moment at which this method is called.
*/
public logExecution(cell: ICell) {
let cellExecution = new CellExecution(cell, new Date());
@ -80,12 +87,15 @@ export class ExecutionLogSlicer {
}
/**
* Use logExecution instead if a cell has just been run. This function is intended to be used
* only to initialize history when a notebook is reloaded.
* Use logExecution instead if a cell has just been run to annotate it with the current time
* as the execution time. This function is intended to be used only to initialize history
* when a notebook is reloaded. However, any method that eventually calls this method will
* notify all observers that this cell has been executed.
*/
public addExecutionToLog(cellExecution: CellExecution) {
this._programBuilder.add(cellExecution.cell);
this._executionLog.push(cellExecution);
this.executionLogged.emit(cellExecution);
}
/**

40
src/main/execution-logger.ts
Просмотреть файл

@ -1,41 +1,17 @@
import { CodeCellModel, ICellModel } from "@jupyterlab/cells";
import { NotebookPanel } from "@jupyterlab/notebook";
import { IObservableList } from "@jupyterlab/observables";
import { GatherModel } from "../model";
import { GatherModel, IGatherObserver, GatherModelEvent, GatherEventData } from "../model";
import { LabCell } from "../model/cell";
export class ExecutionLogger {
export class ExecutionLogger implements IGatherObserver {
constructor(notebook: NotebookPanel, gatherModel: GatherModel) {
constructor(gatherModel: GatherModel) {
gatherModel.addObserver(this);
this._gatherModel = gatherModel;
let existingCells = notebook.content.model.cells;
for (let i = 0; i < existingCells.length; i++) {
this._listenForCellExecution(existingCells.get(i));
}
this._listenToFutureAddedCells(notebook);
}
_listenForCellExecution(cellModel: ICellModel) {
// When a cell is added, register for its state changes.
if (cellModel.type !== 'code') { return; }
cellModel.stateChanged.connect((changedCell, cellStateChange) => {
if (changedCell instanceof CodeCellModel && cellStateChange.name === "executionCount" && cellStateChange.newValue !== undefined && cellStateChange.newValue !== null) {
let cell = new LabCell(changedCell).deepCopy();
this._gatherModel.executionLog.logExecution(cell);
this._gatherModel.lastExecutedCell = cell;
}
});
}
_listenToFutureAddedCells(notebook: NotebookPanel) {
notebook.content.model.cells.changed.connect(
(_, change) => this._onCellsChanged(change));
}
_onCellsChanged(cellListChange: IObservableList.IChangedArgs<ICellModel>): void {
if (cellListChange.type === 'add') {
const cellModel = cellListChange.newValues[0] as ICellModel;
this._listenForCellExecution(cellModel);
public onModelChange(property: GatherModelEvent, eventData: GatherEventData) {
if (property == GatherModelEvent.CELL_EXECUTED) {
let loggableLabCell = (eventData as LabCell).deepCopy();
this._gatherModel.executionLog.logExecution(loggableLabCell);
}
}

15
src/main/gather-actions.ts
Просмотреть файл

@ -6,7 +6,7 @@ import { FileEditor } from "@jupyterlab/fileeditor";
import { INotebookTracker, NotebookPanel } from "@jupyterlab/notebook";
import { Kernel } from "@jupyterlab/services";
import { JSONObject } from "@phosphor/coreutils";
import { ISignal, Signal } from "@phosphor/signaling";
import { Signal } from "@phosphor/signaling";
import { SlicedExecution } from "../analysis/slice/log-slicer";
import { OutputSelection } from "../model";
@ -30,23 +30,24 @@ export class Clipboard {
return this.INSTANCE;
}
get copied(): ISignal<this, SlicedExecution> {
return this._copied;
}
copy(slice: SlicedExecution, outputSelections?: OutputSelection[]) {
const JUPYTER_CELL_MIME = 'application/vnd.jupyter.cells';
if (slice) {
// let cellJson = sliceToCellJson(slice, this._gatherModel.selectedOutputs.concat());
let cellJson = getCellsJsonForSlice(slice, outputSelections);
const clipboard = JupyterClipboard.getInstance();
clipboard.clear();
clipboard.setData(JUPYTER_CELL_MIME, cellJson);
this._copied.emit(slice);
this.copied.emit(slice);
}
}
private static INSTANCE = new Clipboard();
private _copied = new Signal<this, SlicedExecution>(this)
/**
* Signal emitted when a slice is copied to the clipbaord.
*/
readonly copied = new Signal<this, SlicedExecution>(this);
}
/**

11
src/main/main.ts
Просмотреть файл

@ -52,12 +52,17 @@ export class CodeGatheringExtension implements DocumentRegistry.IWidgetExtension
*/
notebookContext.ready.then(() => {
/*
* The order of operations here is key. First, create a model that contains a log of
* executed cells and the state of the gather UI.
*/
let notebookModel = notebookContext.model;
let executionLog = new ExecutionLogSlicer(new DataflowAnalyzer());
let gatherModel = new GatherModel(executionLog);
new ExecutionLogger(gatherModel);
/*
* Initialize reactive UI before loading the execution log from storage. This lets us
* Then,Initialize reactive UI before loading the execution log from storage. This lets us
* update the UI automatically as we populate the log.
*/
this._toolbarWidgets = initToolbar(notebook, gatherModel, this);
@ -65,8 +70,10 @@ export class CodeGatheringExtension implements DocumentRegistry.IWidgetExtension
new CellChangeListener(gatherModel, notebook);
new GatherController(gatherModel, this._documentManager, this._notebooks);
/**
* Then, load the execution log from the notebook's metadata.
*/
this._gatherModelRegistry.addGatherModel(notebookModel, gatherModel);
new ExecutionLogger(notebook, gatherModel);
saveHistoryOnNotebookSave(notebook, gatherModel);
loadHistory(notebookContext.model, gatherModel);
});

58
src/model/cell.ts
Просмотреть файл

@ -25,11 +25,25 @@ export interface ICell {
*/
gathered: boolean;
executionCount: number;
hasError: boolean;
/**
* Whether this cell's text has been changed since its last execution. Undefined behavior when
* a cell has never been executed.
*/
readonly dirty: boolean;
/**
* The cell's current text.
*/
text: string;
executionCount: number;
outputs: nbformat.IOutput[];
/**
* Whether analysis or execution of this cell has yielded an error.
*/
hasError: boolean;
/**
* Flag used for type checking.
*/
@ -53,6 +67,10 @@ export interface ICell {
serialize: () => nbformat.ICodeCell;
}
export function instanceOfICell(object: any): object is ICell {
return object && (typeof object == "object") && "is_cell" in object;
}
/**
* Abstract class for accessing cell data.
*/
@ -69,6 +87,16 @@ export abstract class AbstractCell implements ICell {
abstract outputs: nbformat.IOutput[];
abstract deepCopy(): AbstractCell;
/**
* The cell's text when it was executed, i.e., when the execution count was last changed.
* This will be undefined if the cell has never been executed.
*/
abstract lastExecutedText: string;
get dirty(): boolean {
return this.text !== this.lastExecutedText;
}
/**
* This method is called by the logger to sanitize cell data before logging it. This method
* should elide any sensitive data, like the cell's text.
@ -93,7 +121,7 @@ export abstract class AbstractCell implements ICell {
}
return clone;
});
return new SimpleCell({
return new LogCell({
text: this.text,
hasError: this.hasError,
outputs: clonedOutputs
@ -115,7 +143,10 @@ export abstract class AbstractCell implements ICell {
}
}
export class SimpleCell extends AbstractCell {
/**
* Static cell data. Provides an interfaces to cell data loaded from a log.
*/
export class LogCell extends AbstractCell {
constructor(cellData: {
id?: string, persistentId?: string, executionCount?: number, hasError?: boolean,
@ -128,12 +159,13 @@ export class SimpleCell extends AbstractCell {
this.executionCount = cellData.executionCount || undefined;
this.hasError = cellData.hasError || false;
this.text = cellData.text || "";
this.lastExecutedText = this.text;
this.outputs = cellData.outputs || [];
this.gathered = false;
}
deepCopy(): AbstractCell {
return new SimpleCell(this);
return new LogCell(this);
}
readonly is_cell: boolean;
@ -143,16 +175,14 @@ export class SimpleCell extends AbstractCell {
readonly hasError: boolean;
readonly isCode: boolean;
readonly text: string;
readonly lastExecutedText: string;
readonly outputs: nbformat.IOutput[];
readonly gathered: boolean;
}
export function instanceOfICell(object: any): object is ICell {
return object && (typeof object == "object") && "is_cell" in object;
}
/**
* Abstract interface to data of a Jupyter Lab code cell.
* Wrapper around a code cell model created by Jupyter Lab. Provides a consistent interface to
* lab data to other cells that have been loaded from a log.
*/
export class LabCell extends AbstractCell {
@ -184,6 +214,14 @@ export class LabCell extends AbstractCell {
this._model.value.text = text;
}
get lastExecutedText(): string {
return this._model.metadata.get('last_executed_text') as string;
}
set lastExecutedText(text: string) {
this._model.metadata.set('last_executed_text', text);
}
get executionCount(): number {
return this._model.executionCount;
}

6
src/model/model.ts
Просмотреть файл

@ -22,6 +22,7 @@ export enum GatherState {
export enum GatherModelEvent {
STATE_CHANGED,
CELL_EXECUTED,
CELL_EXECUTION_LOGGED,
CELL_DELETED,
CELL_EDITED,
EDITOR_DEF_FOUND,
@ -56,6 +57,9 @@ export class GatherModel {
constructor(executionLog: ExecutionLogSlicer) {
this._executionLog = executionLog;
this._executionLog.executionLogged.connect((_, cellExecution) => {
this.notifyObservers(GatherModelEvent.CELL_EXECUTION_LOGGED, cellExecution.cell);
});
}
/**
@ -75,7 +79,7 @@ export class GatherModel {
}
/**
* Get execution history for this notebook.
* Get exeuction history for the notebook.
*/
get executionLog(): ExecutionLogSlicer {
return this._executionLog;

21
src/overlay/cell-listener.ts
Просмотреть файл

@ -1,4 +1,4 @@
import { CodeCellModel, ICellModel } from "@jupyterlab/cells";
import { CodeCellModel, ICellModel, ICodeCellModel } from "@jupyterlab/cells";
import { NotebookPanel } from "@jupyterlab/notebook";
import { IObservableList } from "@jupyterlab/observables";
import { GatherModel } from "../model";
@ -11,14 +11,16 @@ export class CellChangeListener {
private _gatherModel: GatherModel;
constructor(gatherModel: GatherModel, panel: NotebookPanel) {
constructor(gatherModel: GatherModel, notebookPanel: NotebookPanel) {
this._gatherModel = gatherModel;
this.registerCurrentCells(notebookPanel);
notebookPanel.content.model.cells.changed.connect((_, change) => this.registerAddedCells(change), this);
}
for (let i = 0; i < panel.content.model.cells.length; i++) {
this.registerCell(panel.content.model.cells.get(i));
private registerCurrentCells(notebookPanel: NotebookPanel) {
for (let i = 0; i < notebookPanel.content.model.cells.length; i++) {
this.registerCell(notebookPanel.content.model.cells.get(i));
}
panel.content.model.cells.changed.connect((_, change) => this.registerAddedCells(change), this);
}
private registerCell(cell: ICellModel) {
@ -27,6 +29,13 @@ export class CellChangeListener {
* A cell will be considered edited whenever any of its contents changed, including
* execution count, metadata, outputs, text, etc.
*/
cell.stateChanged.connect((changedCell, cellStateChange) => {
if (cellStateChange.name === "executionCount" && cellStateChange.newValue !== undefined && cellStateChange.newValue !== null) {
let labCell = new LabCell(changedCell as ICodeCellModel);
labCell.lastExecutedText = labCell.text;
this._gatherModel.lastExecutedCell = labCell;
}
});
cell.contentChanged.connect((changedCell, _) => {
if (changedCell instanceof CodeCellModel) {
this._gatherModel.lastEditedCell = new LabCell(changedCell);

44
src/overlay/gather-markers.ts
Просмотреть файл

@ -3,13 +3,14 @@
*/
import { NotebookPanel } from "@jupyterlab/notebook";
import { LineHandle } from "codemirror";
import { ICell } from "../model/cell";
import { ICell, LabCell } from "../model/cell";
import { ILocation, ISyntaxNode } from "../analysis/parse/python/python-parser";
import { Ref, SymbolType } from "../analysis/slice/data-flow";
import { SlicedExecution } from "../analysis/slice/log-slicer";
import { log } from "../util/log";
import { CellOutput, DefSelection, EditorDef, GatherEventData, GatherModel, GatherModelEvent, IGatherObserver, OutputSelection } from "../model";
import { NotebookElementFinder } from "./element-finder";
import { ICodeCellModel } from "@jupyterlab/cells";
/**
* Class for a highlighted, clickable output.
@ -37,10 +38,15 @@ const DEFINITION_SELECTED_CLASS = "jp-InputArea-editor-nametext-selected";
const DEFINITION_LINE_SELECTED_CLASS = "jp-InputArea-editor-nameline-selected";
/**
* Class for a data dependency.
* Class for a line with a data dependency.
*/
const DEPENDENCY_CLASS = "jp-InputArea-editor-dependencyline";
/**
* Class for a line with a data dependency in a dirty cell.
*/
const DIRTY_DEPENDENCY_CLASS = "jp-InputArea-editor-dirtydependencyline";
/**
* Clear existing selections in the window.
*/
@ -96,7 +102,7 @@ export class MarkerManager implements IGatherObserver {
onModelChange(eventType: GatherModelEvent, eventData: GatherEventData, model: GatherModel) {
// When a cell is executed, search for definitions and output.
if (eventType == GatherModelEvent.CELL_EXECUTED) {
if (eventType == GatherModelEvent.CELL_EXECUTION_LOGGED) {
let cell = eventData as ICell;
this.clearSelectablesForCell(cell);
let editor = this._elementFinder.getEditor(cell);
@ -110,6 +116,7 @@ export class MarkerManager implements IGatherObserver {
// When a cell is deleted or edited, delete all of its def markers.
if (eventType == GatherModelEvent.CELL_DELETED || eventType == GatherModelEvent.CELL_EDITED) {
let cell = eventData as ICell;
this._updateDependenceHighlightsForCell(cell);
this.clearSelectablesForCell(cell);
}
@ -295,18 +302,21 @@ export class MarkerManager implements IGatherObserver {
highlightDependencies(slice: SlicedExecution) {
let defLines: number[] = [];
slice.cellSlices.forEach(cellSlice => {
let cell = cellSlice.cell;
let loggedCell = cellSlice.cell;
let sliceLocations = cellSlice.slice;
let editor = this._elementFinder.getEditorWithExecutionCount(cell);
let liveCellWidget = this._elementFinder.getCell(loggedCell.persistentId, loggedCell.executionCount);
let editor = this._elementFinder.getEditorWithExecutionCount(loggedCell);
if (editor) {
if (liveCellWidget && editor) {
let liveCell = new LabCell(liveCellWidget.model as ICodeCellModel);
let numLines = 0;
// Batch the highlight operations for each cell to spend less time updating cell height.
editor.operation(() => {
sliceLocations.items.forEach((loc:ILocation) => {
for (let lineNumber = loc.first_line - 1; lineNumber <= loc.last_line -1; lineNumber++) {
numLines += 1;
let lineHandle = editor.addLineClass(lineNumber, "background", DEPENDENCY_CLASS);
let styleClass = liveCell.dirty ? DIRTY_DEPENDENCY_CLASS : DEPENDENCY_CLASS;
let lineHandle = editor.addLineClass(lineNumber, "background", styleClass);
this._dependencyLineMarkers.push({ editor: editor, lineHandle: lineHandle });
}
});
@ -317,10 +327,28 @@ export class MarkerManager implements IGatherObserver {
log("Added lines for defs (may be overlapping)", { defLines });
}
private _clearDependencyMarkersForLine(editor: CodeMirror.Editor, lineHandle: CodeMirror.LineHandle) {
editor.removeLineClass(lineHandle, "background", DEPENDENCY_CLASS);
editor.removeLineClass(lineHandle, "background", DIRTY_DEPENDENCY_CLASS);
}
private _updateDependenceHighlightsForCell(cell: ICell) {
let editor = this._elementFinder.getEditorWithExecutionCount(cell);
let liveCellWidget = this._elementFinder.getCell(cell.persistentId, cell.executionCount);
let liveCell = new LabCell(liveCellWidget.model as ICodeCellModel);
this._dependencyLineMarkers
.filter((marker) => marker.editor == editor)
.forEach((marker) => {
this._clearDependencyMarkersForLine(marker.editor, marker.lineHandle);
let styleClass = liveCell.dirty ? DIRTY_DEPENDENCY_CLASS : DEPENDENCY_CLASS;
marker.editor.addLineClass(marker.lineHandle, "background", styleClass);
});
}
private _clearDependencyLineMarkers() {
log("Cleared all dependency line markers");
this._dependencyLineMarkers.forEach(marker => {
marker.editor.removeLineClass(marker.lineHandle, "background", DEPENDENCY_CLASS);
this._clearDependencyMarkersForLine(marker.editor, marker.lineHandle);
})
this._dependencyLineMarkers = [];
}

5
src/persistence/load.ts
Просмотреть файл

@ -2,7 +2,7 @@ import { INotebookModel } from "@jupyterlab/notebook";
import { JSONArray, JSONExt, JSONObject } from "@phosphor/coreutils";
import { log } from "util";
import { CellExecution } from "../analysis/slice/log-slicer";
import { SimpleCell } from "../model/cell";
import { LogCell } from "../model/cell";
import { GatherModel } from "../model";
/**
@ -50,7 +50,6 @@ function _tryLoadHistory(notebookModel: INotebookModel, gatherModel: GatherModel
return;
}
gatherModel.executionLog.addExecutionToLog(cellExecution);
gatherModel.lastExecutedCell = cellExecution.cell;
}
}
@ -107,6 +106,6 @@ function _loadExecutionFromJson(executionJson: JSONObject): CellExecution {
/**
* TODO(andrewhead): Update with Kunal's code for serializing and deserializing outputs.
*/
let cell = new SimpleCell({ id, executionCount, hasError, text, persistentId });
let cell = new LogCell({ id, executionCount, hasError, text, persistentId });
return new CellExecution(cell, executionTime);
}

6
src/test/cellslice.test.ts
Просмотреть файл

@ -1,12 +1,12 @@
import { expect } from 'chai';
import { LocationSet } from '../analysis/slice/slice';
import { SimpleCell } from '../model/cell';
import { LogCell } from '../model/cell';
import { CellSlice } from '../model/cellslice';
describe('CellSlice', () => {
it('yields a text slice based on a set of locations', () => {
let cellSlice = new CellSlice(new SimpleCell({
let cellSlice = new CellSlice(new LogCell({
text: [
"a = 1",
"b = 2",
@ -27,7 +27,7 @@ describe('CellSlice', () => {
});
it('yields entire lines if requested', () => {
let cellSlice = new CellSlice(new SimpleCell({
let cellSlice = new CellSlice(new LogCell({
text: [
"a = 1",
"b = 2",

4
src/test/merge.test.ts
Просмотреть файл

@ -1,13 +1,13 @@
import { expect } from "chai";
import { SlicedExecution } from "../analysis/slice/log-slicer";
import { LocationSet } from "../analysis/slice/slice";
import { ICell, SimpleCell } from "../model/cell";
import { ICell, LogCell } from "../model/cell";
import { CellSlice } from "../model/cellslice";
describe('SlicedExecution', () => {
function cell(persistentId: string, executionCount: number, ...codeLines: string[]): ICell {
return new SimpleCell({ executionCount, text: codeLines.join("\n"), persistentId });
return new LogCell({ executionCount, text: codeLines.join("\n"), persistentId });
}
function cellSlice(cell: ICell, slice: LocationSet): CellSlice {

4
src/test/program-builder.test.ts
Просмотреть файл

@ -1,13 +1,13 @@
import { expect } from "chai";
import { ProgramBuilder } from "../analysis/slice/program-builder";
import { ICell, SimpleCell } from '../model/cell';
import { ICell, LogCell } from '../model/cell';
describe('program builder', () => {
function createCell(persistentId: string, executionCount: number, ...codeLines: string[]): ICell {
let text = codeLines.join("\n");
return new SimpleCell({ executionCount, persistentId, text });
return new LogCell({ executionCount, persistentId, text });
}
let programBuilder: ProgramBuilder;

7
style/index.css
Просмотреть файл

@ -92,9 +92,12 @@ span.jp-GatherLabel {
font-weight: bold;
}
div.CodeMirror-linebackground.jp-InputArea-editor-dirtydependencyline {
background-color: #f7d3cf;
}
div.CodeMirror-linebackground.jp-InputArea-editor-dependencyline {
/* background-color: var(--brand-color4); */
background-color: #f4eafc;
background-color: #e6d4f4;
}
/**