initial commit.
This commit is contained in:
Sebastian Burckhardt
Коммит
0c8e28c4d4
|
|
@ -0,0 +1,198 @@
|
|||
# Remove the line below if you want to inherit .editorconfig settings from higher directories
|
||||
root = true
|
||||
|
||||
# C# files
|
||||
[*.cs]
|
||||
|
||||
#### Core EditorConfig Options ####
|
||||
|
||||
# Indentation and spacing
|
||||
indent_size = 4
|
||||
indent_style = space
|
||||
tab_width = 4
|
||||
|
||||
# New line preferences
|
||||
end_of_line = crlf
|
||||
insert_final_newline = false
|
||||
|
||||
#### .NET Coding Conventions ####
|
||||
|
||||
# Organize usings
|
||||
dotnet_separate_import_directive_groups = false
|
||||
dotnet_sort_system_directives_first = true
|
||||
|
||||
# this. and Me. preferences
|
||||
dotnet_style_qualification_for_event = true:warning
|
||||
dotnet_style_qualification_for_field = true:warning
|
||||
dotnet_style_qualification_for_method = true:warning
|
||||
dotnet_style_qualification_for_property = true:warning
|
||||
|
||||
# Language keywords vs BCL types preferences
|
||||
dotnet_style_predefined_type_for_locals_parameters_members = true:warning
|
||||
dotnet_style_predefined_type_for_member_access = true:silent
|
||||
|
||||
# Parentheses preferences
|
||||
dotnet_style_parentheses_in_arithmetic_binary_operators = always_for_clarity:silent
|
||||
dotnet_style_parentheses_in_other_binary_operators = always_for_clarity:silent
|
||||
dotnet_style_parentheses_in_other_operators = never_if_unnecessary:silent
|
||||
dotnet_style_parentheses_in_relational_binary_operators = always_for_clarity:silent
|
||||
|
||||
# Modifier preferences
|
||||
dotnet_style_require_accessibility_modifiers = omit_if_default:warning
|
||||
|
||||
# Expression-level preferences
|
||||
dotnet_style_coalesce_expression = true:suggestion
|
||||
dotnet_style_collection_initializer = true:suggestion
|
||||
dotnet_style_explicit_tuple_names = true:suggestion
|
||||
dotnet_style_null_propagation = true:suggestion
|
||||
dotnet_style_object_initializer = true:suggestion
|
||||
dotnet_style_prefer_auto_properties = true:silent
|
||||
dotnet_style_prefer_compound_assignment = true:suggestion
|
||||
dotnet_style_prefer_conditional_expression_over_assignment = true:silent
|
||||
dotnet_style_prefer_conditional_expression_over_return = true:silent
|
||||
dotnet_style_prefer_inferred_anonymous_type_member_names = true:suggestion
|
||||
dotnet_style_prefer_inferred_tuple_names = true:suggestion
|
||||
dotnet_style_prefer_is_null_check_over_reference_equality_method = true:suggestion
|
||||
dotnet_style_prefer_simplified_interpolation = true:suggestion
|
||||
|
||||
# Field preferences
|
||||
dotnet_style_readonly_field = true:warning
|
||||
|
||||
# Parameter preferences
|
||||
dotnet_code_quality_unused_parameters = all:suggestion
|
||||
|
||||
#### C# Coding Conventions ####
|
||||
|
||||
# var preferences
|
||||
csharp_style_var_elsewhere = false:suggestion
|
||||
csharp_style_var_for_built_in_types = false:suggestion
|
||||
csharp_style_var_when_type_is_apparent = false:silent
|
||||
|
||||
# Expression-bodied members
|
||||
csharp_style_expression_bodied_accessors = true:silent
|
||||
csharp_style_expression_bodied_constructors = false:silent
|
||||
csharp_style_expression_bodied_indexers = true:silent
|
||||
csharp_style_expression_bodied_lambdas = true:silent
|
||||
csharp_style_expression_bodied_local_functions = false:silent
|
||||
csharp_style_expression_bodied_methods = false:silent
|
||||
csharp_style_expression_bodied_operators = false:silent
|
||||
csharp_style_expression_bodied_properties = true:silent
|
||||
|
||||
# Pattern matching preferences
|
||||
csharp_style_pattern_matching_over_as_with_null_check = true:suggestion
|
||||
csharp_style_pattern_matching_over_is_with_cast_check = true:suggestion
|
||||
csharp_style_prefer_switch_expression = false:suggestion
|
||||
|
||||
# Null-checking preferences
|
||||
csharp_style_conditional_delegate_call = true:suggestion
|
||||
|
||||
# Modifier preferences
|
||||
csharp_prefer_static_local_function = true:suggestion
|
||||
csharp_preferred_modifier_order = public,private,protected,internal,static,extern,new,virtual,abstract,sealed,override,readonly,unsafe,volatile,async:silent
|
||||
|
||||
# Code-block preferences
|
||||
csharp_prefer_braces = true:suggestion
|
||||
csharp_prefer_simple_using_statement = true:suggestion
|
||||
|
||||
# Expression-level preferences
|
||||
csharp_prefer_simple_default_expression = true:suggestion
|
||||
csharp_style_deconstructed_variable_declaration = true:suggestion
|
||||
csharp_style_inlined_variable_declaration = true:suggestion
|
||||
csharp_style_pattern_local_over_anonymous_function = true:suggestion
|
||||
csharp_style_prefer_index_operator = true:suggestion
|
||||
csharp_style_prefer_range_operator = true:suggestion
|
||||
csharp_style_throw_expression = true:suggestion
|
||||
csharp_style_unused_value_assignment_preference = discard_variable:suggestion
|
||||
csharp_style_unused_value_expression_statement_preference = discard_variable:silent
|
||||
|
||||
# 'using' directive preferences
|
||||
csharp_using_directive_placement = inside_namespace:warning
|
||||
|
||||
#### C# Formatting Rules ####
|
||||
|
||||
# New line preferences
|
||||
csharp_new_line_before_catch = true
|
||||
csharp_new_line_before_else = true
|
||||
csharp_new_line_before_finally = true
|
||||
csharp_new_line_before_members_in_anonymous_types = true
|
||||
csharp_new_line_before_members_in_object_initializers = true
|
||||
csharp_new_line_before_open_brace = all
|
||||
csharp_new_line_between_query_expression_clauses = true
|
||||
|
||||
# Indentation preferences
|
||||
csharp_indent_block_contents = true
|
||||
csharp_indent_braces = false
|
||||
csharp_indent_case_contents = true
|
||||
csharp_indent_case_contents_when_block = true
|
||||
csharp_indent_labels = one_less_than_current
|
||||
csharp_indent_switch_labels = true
|
||||
|
||||
# Space preferences
|
||||
csharp_space_after_cast = false
|
||||
csharp_space_after_colon_in_inheritance_clause = true
|
||||
csharp_space_after_comma = true
|
||||
csharp_space_after_dot = false
|
||||
csharp_space_after_keywords_in_control_flow_statements = true
|
||||
csharp_space_after_semicolon_in_for_statement = true
|
||||
csharp_space_around_binary_operators = before_and_after
|
||||
csharp_space_around_declaration_statements = false
|
||||
csharp_space_before_colon_in_inheritance_clause = true
|
||||
csharp_space_before_comma = false
|
||||
csharp_space_before_dot = false
|
||||
csharp_space_before_open_square_brackets = false
|
||||
csharp_space_before_semicolon_in_for_statement = false
|
||||
csharp_space_between_empty_square_brackets = false
|
||||
csharp_space_between_method_call_empty_parameter_list_parentheses = false
|
||||
csharp_space_between_method_call_name_and_opening_parenthesis = false
|
||||
csharp_space_between_method_call_parameter_list_parentheses = false
|
||||
csharp_space_between_method_declaration_empty_parameter_list_parentheses = false
|
||||
csharp_space_between_method_declaration_name_and_open_parenthesis = false
|
||||
csharp_space_between_method_declaration_parameter_list_parentheses = false
|
||||
csharp_space_between_parentheses = false
|
||||
csharp_space_between_square_brackets = false
|
||||
|
||||
# Wrapping preferences
|
||||
csharp_preserve_single_line_blocks = true
|
||||
csharp_preserve_single_line_statements = true
|
||||
|
||||
#### Naming styles ####
|
||||
|
||||
# Naming rules
|
||||
|
||||
dotnet_naming_rule.interface_should_be_begins_with_i.severity = suggestion
|
||||
dotnet_naming_rule.interface_should_be_begins_with_i.symbols = interface
|
||||
dotnet_naming_rule.interface_should_be_begins_with_i.style = begins_with_i
|
||||
|
||||
dotnet_naming_rule.types_should_be_pascal_case.severity = suggestion
|
||||
dotnet_naming_rule.types_should_be_pascal_case.symbols = types
|
||||
dotnet_naming_rule.types_should_be_pascal_case.style = pascal_case
|
||||
|
||||
dotnet_naming_rule.non_field_members_should_be_pascal_case.severity = suggestion
|
||||
dotnet_naming_rule.non_field_members_should_be_pascal_case.symbols = non_field_members
|
||||
dotnet_naming_rule.non_field_members_should_be_pascal_case.style = pascal_case
|
||||
|
||||
# Symbol specifications
|
||||
|
||||
dotnet_naming_symbols.interface.applicable_kinds = interface
|
||||
dotnet_naming_symbols.interface.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected
|
||||
dotnet_naming_symbols.interface.required_modifiers =
|
||||
|
||||
dotnet_naming_symbols.types.applicable_kinds = class, struct, interface, enum
|
||||
dotnet_naming_symbols.types.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected
|
||||
dotnet_naming_symbols.types.required_modifiers =
|
||||
|
||||
dotnet_naming_symbols.non_field_members.applicable_kinds = property, event, method
|
||||
dotnet_naming_symbols.non_field_members.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected
|
||||
dotnet_naming_symbols.non_field_members.required_modifiers =
|
||||
|
||||
# Naming styles
|
||||
|
||||
dotnet_naming_style.pascal_case.required_prefix =
|
||||
dotnet_naming_style.pascal_case.required_suffix =
|
||||
dotnet_naming_style.pascal_case.word_separator =
|
||||
dotnet_naming_style.pascal_case.capitalization = pascal_case
|
||||
|
||||
dotnet_naming_style.begins_with_i.required_prefix = I
|
||||
dotnet_naming_style.begins_with_i.required_suffix =
|
||||
dotnet_naming_style.begins_with_i.word_separator =
|
||||
dotnet_naming_style.begins_with_i.capitalization = pascal_case
|
||||
|
|
@ -0,0 +1,439 @@
|
|||
## Ignore Visual Studio temporary files, build results, and
|
||||
## files generated by popular Visual Studio add-ons.
|
||||
##
|
||||
## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore
|
||||
|
||||
# User-specific files
|
||||
*.rsuser
|
||||
*.suo
|
||||
*.user
|
||||
*.userosscache
|
||||
*.sln.docstates
|
||||
|
||||
# User-specific files (MonoDevelop/Xamarin Studio)
|
||||
*.userprefs
|
||||
|
||||
# Mono auto generated files
|
||||
mono_crash.*
|
||||
|
||||
# Build results
|
||||
[Dd]ebug/
|
||||
[Dd]ebugPublic/
|
||||
[Rr]elease/
|
||||
[Rr]eleases/
|
||||
x64/
|
||||
x86/
|
||||
[Aa][Rr][Mm]/
|
||||
[Aa][Rr][Mm]64/
|
||||
bld/
|
||||
[Bb]in/
|
||||
[Oo]bj/
|
||||
[Ll]og/
|
||||
|
||||
# Visual Studio 2015/2017 cache/options directory
|
||||
.vs/
|
||||
# Uncomment if you have tasks that create the project's static files in wwwroot
|
||||
#wwwroot/
|
||||
|
||||
# Visual Studio 2017 auto generated files
|
||||
Generated\ Files/
|
||||
|
||||
# MSTest test Results
|
||||
[Tt]est[Rr]esult*/
|
||||
[Bb]uild[Ll]og.*
|
||||
|
||||
# NUNIT
|
||||
*.VisualState.xml
|
||||
TestResult.xml
|
||||
|
||||
# Build Results of an ATL Project
|
||||
[Dd]ebugPS/
|
||||
[Rr]eleasePS/
|
||||
dlldata.c
|
||||
|
||||
# Benchmark Results
|
||||
BenchmarkDotNet.Artifacts/
|
||||
|
||||
# .NET Core
|
||||
project.lock.json
|
||||
project.fragment.lock.json
|
||||
artifacts/
|
||||
|
||||
# StyleCop
|
||||
StyleCopReport.xml
|
||||
|
||||
# Files built by Visual Studio
|
||||
*_i.c
|
||||
*_p.c
|
||||
*_h.h
|
||||
*.ilk
|
||||
*.meta
|
||||
*.obj
|
||||
*.iobj
|
||||
*.pch
|
||||
*.pdb
|
||||
*.ipdb
|
||||
*.pgc
|
||||
*.pgd
|
||||
*.rsp
|
||||
*.sbr
|
||||
*.tlb
|
||||
*.tli
|
||||
*.tlh
|
||||
*.tmp
|
||||
*.tmp_proj
|
||||
*_wpftmp.csproj
|
||||
*.log
|
||||
*.vspscc
|
||||
*.vssscc
|
||||
.builds
|
||||
*.pidb
|
||||
*.svclog
|
||||
*.scc
|
||||
|
||||
# Chutzpah Test files
|
||||
_Chutzpah*
|
||||
|
||||
# Visual C++ cache files
|
||||
ipch/
|
||||
*.aps
|
||||
*.ncb
|
||||
*.opendb
|
||||
*.opensdf
|
||||
*.sdf
|
||||
*.cachefile
|
||||
*.VC.db
|
||||
*.VC.VC.opendb
|
||||
|
||||
# Visual Studio profiler
|
||||
*.psess
|
||||
*.vsp
|
||||
*.vspx
|
||||
*.sap
|
||||
|
||||
# Visual Studio Trace Files
|
||||
*.e2e
|
||||
|
||||
# TFS 2012 Local Workspace
|
||||
$tf/
|
||||
|
||||
# Guidance Automation Toolkit
|
||||
*.gpState
|
||||
|
||||
# ReSharper is a .NET coding add-in
|
||||
_ReSharper*/
|
||||
*.[Rr]e[Ss]harper
|
||||
*.DotSettings.user
|
||||
|
||||
# JustCode is a .NET coding add-in
|
||||
.JustCode
|
||||
|
||||
# TeamCity is a build add-in
|
||||
_TeamCity*
|
||||
|
||||
# DotCover is a Code Coverage Tool
|
||||
*.dotCover
|
||||
|
||||
# AxoCover is a Code Coverage Tool
|
||||
.axoCover/*
|
||||
!.axoCover/settings.json
|
||||
|
||||
# Visual Studio code coverage results
|
||||
*.coverage
|
||||
*.coveragexml
|
||||
|
||||
# NCrunch
|
||||
_NCrunch_*
|
||||
.*crunch*.local.xml
|
||||
nCrunchTemp_*
|
||||
|
||||
# MightyMoose
|
||||
*.mm.*
|
||||
AutoTest.Net/
|
||||
|
||||
# Web workbench (sass)
|
||||
.sass-cache/
|
||||
|
||||
# Installshield output folder
|
||||
[Ee]xpress/
|
||||
|
||||
# DocProject is a documentation generator add-in
|
||||
DocProject/buildhelp/
|
||||
DocProject/Help/*.HxT
|
||||
DocProject/Help/*.HxC
|
||||
DocProject/Help/*.hhc
|
||||
DocProject/Help/*.hhk
|
||||
DocProject/Help/*.hhp
|
||||
DocProject/Help/Html2
|
||||
DocProject/Help/html
|
||||
|
||||
# Click-Once directory
|
||||
publish/
|
||||
|
||||
# Publish Web Output
|
||||
*.[Pp]ublish.xml
|
||||
*.azurePubxml
|
||||
# Note: Comment the next line if you want to checkin your web deploy settings,
|
||||
# but database connection strings (with potential passwords) will be unencrypted
|
||||
*.pubxml
|
||||
*.publishproj
|
||||
|
||||
# Microsoft Azure Web App publish settings. Comment the next line if you want to
|
||||
# checkin your Azure Web App publish settings, but sensitive information contained
|
||||
# in these scripts will be unencrypted
|
||||
PublishScripts/
|
||||
|
||||
# NuGet Packages
|
||||
*.nupkg
|
||||
# The packages folder can be ignored because of Package Restore
|
||||
**/[Pp]ackages/*
|
||||
# except build/, which is used as an MSBuild target.
|
||||
!**/[Pp]ackages/build/
|
||||
# Uncomment if necessary however generally it will be regenerated when needed
|
||||
#!**/[Pp]ackages/repositories.config
|
||||
# NuGet v3's project.json files produces more ignorable files
|
||||
*.nuget.props
|
||||
*.nuget.targets
|
||||
|
||||
# Microsoft Azure Build Output
|
||||
csx/
|
||||
*.build.csdef
|
||||
|
||||
# Microsoft Azure Emulator
|
||||
ecf/
|
||||
rcf/
|
||||
|
||||
# Windows Store app package directories and files
|
||||
AppPackages/
|
||||
BundleArtifacts/
|
||||
Package.StoreAssociation.xml
|
||||
_pkginfo.txt
|
||||
*.appx
|
||||
*.appxbundle
|
||||
*.appxupload
|
||||
|
||||
# Visual Studio cache files
|
||||
# files ending in .cache can be ignored
|
||||
*.[Cc]ache
|
||||
# but keep track of directories ending in .cache
|
||||
!?*.[Cc]ache/
|
||||
|
||||
# Others
|
||||
ClientBin/
|
||||
~$*
|
||||
*~
|
||||
*.dbmdl
|
||||
*.dbproj.schemaview
|
||||
*.jfm
|
||||
*.pfx
|
||||
*.publishsettings
|
||||
orleans.codegen.cs
|
||||
|
||||
# Including strong name files can present a security risk
|
||||
# (https://github.com/github/gitignore/pull/2483#issue-259490424)
|
||||
#*.snk
|
||||
|
||||
# Since there are multiple workflows, uncomment next line to ignore bower_components
|
||||
# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
|
||||
#bower_components/
|
||||
|
||||
# RIA/Silverlight projects
|
||||
Generated_Code/
|
||||
|
||||
# Backup & report files from converting an old project file
|
||||
# to a newer Visual Studio version. Backup files are not needed,
|
||||
# because we have git ;-)
|
||||
_UpgradeReport_Files/
|
||||
Backup*/
|
||||
UpgradeLog*.XML
|
||||
UpgradeLog*.htm
|
||||
ServiceFabricBackup/
|
||||
*.rptproj.bak
|
||||
|
||||
# SQL Server files
|
||||
*.mdf
|
||||
*.ldf
|
||||
*.ndf
|
||||
|
||||
# Business Intelligence projects
|
||||
*.rdl.data
|
||||
*.bim.layout
|
||||
*.bim_*.settings
|
||||
*.rptproj.rsuser
|
||||
*- Backup*.rdl
|
||||
|
||||
# Microsoft Fakes
|
||||
FakesAssemblies/
|
||||
|
||||
# GhostDoc plugin setting file
|
||||
*.GhostDoc.xml
|
||||
|
||||
# Node.js Tools for Visual Studio
|
||||
.ntvs_analysis.dat
|
||||
node_modules/
|
||||
|
||||
# Visual Studio 6 build log
|
||||
*.plg
|
||||
|
||||
# Visual Studio 6 workspace options file
|
||||
*.opt
|
||||
|
||||
# Visual Studio 6 auto-generated workspace file (contains which files were open etc.)
|
||||
*.vbw
|
||||
|
||||
# Visual Studio LightSwitch build output
|
||||
**/*.HTMLClient/GeneratedArtifacts
|
||||
**/*.DesktopClient/GeneratedArtifacts
|
||||
**/*.DesktopClient/ModelManifest.xml
|
||||
**/*.Server/GeneratedArtifacts
|
||||
**/*.Server/ModelManifest.xml
|
||||
_Pvt_Extensions
|
||||
|
||||
# Paket dependency manager
|
||||
.paket/paket.exe
|
||||
paket-files/
|
||||
|
||||
# FAKE - F# Make
|
||||
.fake/
|
||||
|
||||
# CodeRush personal settings
|
||||
.cr/personal
|
||||
|
||||
# Python Tools for Visual Studio (PTVS)
|
||||
__pycache__/
|
||||
*.pyc
|
||||
|
||||
# Cake - Uncomment if you are using it
|
||||
# tools/**
|
||||
# !tools/packages.config
|
||||
|
||||
# Tabs Studio
|
||||
*.tss
|
||||
|
||||
# Telerik's JustMock configuration file
|
||||
*.jmconfig
|
||||
|
||||
# BizTalk build output
|
||||
*.btp.cs
|
||||
*.btm.cs
|
||||
*.odx.cs
|
||||
*.xsd.cs
|
||||
|
||||
# OpenCover UI analysis results
|
||||
OpenCover/
|
||||
|
||||
# Azure Stream Analytics local run output
|
||||
ASALocalRun/
|
||||
|
||||
# MSBuild Binary and Structured Log
|
||||
*.binlog
|
||||
|
||||
# NVidia Nsight GPU debugger configuration file
|
||||
*.nvuser
|
||||
|
||||
# MFractors (Xamarin productivity tool) working folder
|
||||
.mfractor/
|
||||
|
||||
# Local History for Visual Studio
|
||||
.localhistory/
|
||||
|
||||
# BeatPulse healthcheck temp database
|
||||
healthchecksdb
|
||||
|
||||
# Backup folder for Package Reference Convert tool in Visual Studio 2017
|
||||
MigrationBackup/
|
||||
|
||||
##
|
||||
## Visual studio for Mac
|
||||
##
|
||||
|
||||
|
||||
# globs
|
||||
Makefile.in
|
||||
*.userprefs
|
||||
*.usertasks
|
||||
config.make
|
||||
config.status
|
||||
aclocal.m4
|
||||
install-sh
|
||||
autom4te.cache/
|
||||
*.tar.gz
|
||||
tarballs/
|
||||
test-results/
|
||||
|
||||
# Mac bundle stuff
|
||||
*.dmg
|
||||
*.app
|
||||
|
||||
# content below from: https://github.com/github/gitignore/blob/master/Global/macOS.gitignore
|
||||
# General
|
||||
.DS_Store
|
||||
.AppleDouble
|
||||
.LSOverride
|
||||
|
||||
# Icon must end with two \r
|
||||
Icon
|
||||
|
||||
|
||||
# Thumbnails
|
||||
._*
|
||||
|
||||
# Files that might appear in the root of a volume
|
||||
.DocumentRevisions-V100
|
||||
.fseventsd
|
||||
.Spotlight-V100
|
||||
.TemporaryItems
|
||||
.Trashes
|
||||
.VolumeIcon.icns
|
||||
.com.apple.timemachine.donotpresent
|
||||
|
||||
# Directories potentially created on remote AFP share
|
||||
.AppleDB
|
||||
.AppleDesktop
|
||||
Network Trash Folder
|
||||
Temporary Items
|
||||
.apdisk
|
||||
|
||||
# content below from: https://github.com/github/gitignore/blob/master/Global/Windows.gitignore
|
||||
# Windows thumbnail cache files
|
||||
Thumbs.db
|
||||
ehthumbs.db
|
||||
ehthumbs_vista.db
|
||||
|
||||
# Dump file
|
||||
*.stackdump
|
||||
|
||||
# Folder config file
|
||||
[Dd]esktop.ini
|
||||
|
||||
# Recycle Bin used on file shares
|
||||
$RECYCLE.BIN/
|
||||
|
||||
# Windows Installer files
|
||||
*.cab
|
||||
*.msi
|
||||
*.msix
|
||||
*.msm
|
||||
*.msp
|
||||
|
||||
# Windows shortcuts
|
||||
*.lnk
|
||||
|
||||
# JetBrains Rider
|
||||
.idea/
|
||||
*.sln.iml
|
||||
|
||||
##
|
||||
## Visual Studio Code
|
||||
##
|
||||
.vscode/*
|
||||
!.vscode/settings.json
|
||||
!.vscode/tasks.json
|
||||
!.vscode/launch.json
|
||||
!.vscode/extensions.json
|
||||
|
||||
# Other
|
||||
.ionide/
|
||||
|
||||
# The build keeps auto-generating this file. We do not want this.
|
||||
/.nuget/NuGet.Config
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
|
||||
Microsoft Visual Studio Solution File, Format Version 12.00
|
||||
# Visual Studio Version 16
|
||||
VisualStudioVersion = 16.0.29806.167
|
||||
MinimumVisualStudioVersion = 10.0.40219.1
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DurableTask.Netherite", "src\DurableTask.Netherite\DurableTask.Netherite.csproj", "{FC7FB0AF-2322-4356-AF64-A8E2EB7D1EF8}"
|
||||
EndProject
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DurableTask.Netherite.Tests", "test\DurableTask.Netherite.Tests\DurableTask.Netherite.Tests.csproj", "{3D282458-56C8-4022-A37C-6B67F4EF9BF4}"
|
||||
EndProject
|
||||
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{FFF00AD0-D467-4D12-AB79-BEF19BA527C0}"
|
||||
ProjectSection(SolutionItems) = preProject
|
||||
.editorconfig = .editorconfig
|
||||
.gitignore = .gitignore
|
||||
nuget.config = nuget.config
|
||||
sign.snk = sign.snk
|
||||
EndProjectSection
|
||||
EndProject
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DurableTask.Netherite.AzureFunctions", "src\DurableTask.Netherite.AzureFunctions\DurableTask.Netherite.AzureFunctions.csproj", "{FC03BDA8-8A73-45F4-8D21-25F739BFF9E1}"
|
||||
EndProject
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DurableTask.Netherite.AzureFunctions.Tests", "test\DurableTask.Netherite.AzureFunctions.Tests\DurableTask.Netherite.AzureFunctions.Tests.csproj", "{1809EEF7-0772-404A-96C2-D76D80F1D191}"
|
||||
EndProject
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PerformanceTests", "test\PerformanceTests\PerformanceTests.csproj", "{DD1E1B3F-4FA2-4F3A-9AE1-6B2A0B864AAF}"
|
||||
EndProject
|
||||
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{D33AB157-04B9-4BAD-B580-C3C87C17828C}"
|
||||
EndProject
|
||||
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{4A7226CF-57BF-4CA3-A4AC-91A398A1D84B}"
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
Debug|Any CPU = Debug|Any CPU
|
||||
Release|Any CPU = Release|Any CPU
|
||||
EndGlobalSection
|
||||
GlobalSection(ProjectConfigurationPlatforms) = postSolution
|
||||
{FC7FB0AF-2322-4356-AF64-A8E2EB7D1EF8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{FC7FB0AF-2322-4356-AF64-A8E2EB7D1EF8}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{FC7FB0AF-2322-4356-AF64-A8E2EB7D1EF8}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{FC7FB0AF-2322-4356-AF64-A8E2EB7D1EF8}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{3D282458-56C8-4022-A37C-6B67F4EF9BF4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{3D282458-56C8-4022-A37C-6B67F4EF9BF4}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{3D282458-56C8-4022-A37C-6B67F4EF9BF4}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{3D282458-56C8-4022-A37C-6B67F4EF9BF4}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{FC03BDA8-8A73-45F4-8D21-25F739BFF9E1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{FC03BDA8-8A73-45F4-8D21-25F739BFF9E1}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{FC03BDA8-8A73-45F4-8D21-25F739BFF9E1}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{FC03BDA8-8A73-45F4-8D21-25F739BFF9E1}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{1809EEF7-0772-404A-96C2-D76D80F1D191}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{1809EEF7-0772-404A-96C2-D76D80F1D191}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{1809EEF7-0772-404A-96C2-D76D80F1D191}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{1809EEF7-0772-404A-96C2-D76D80F1D191}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{DD1E1B3F-4FA2-4F3A-9AE1-6B2A0B864AAF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{DD1E1B3F-4FA2-4F3A-9AE1-6B2A0B864AAF}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{DD1E1B3F-4FA2-4F3A-9AE1-6B2A0B864AAF}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{DD1E1B3F-4FA2-4F3A-9AE1-6B2A0B864AAF}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
EndGlobalSection
|
||||
GlobalSection(SolutionProperties) = preSolution
|
||||
HideSolutionNode = FALSE
|
||||
EndGlobalSection
|
||||
GlobalSection(NestedProjects) = preSolution
|
||||
{FC7FB0AF-2322-4356-AF64-A8E2EB7D1EF8} = {D33AB157-04B9-4BAD-B580-C3C87C17828C}
|
||||
{3D282458-56C8-4022-A37C-6B67F4EF9BF4} = {4A7226CF-57BF-4CA3-A4AC-91A398A1D84B}
|
||||
{FC03BDA8-8A73-45F4-8D21-25F739BFF9E1} = {D33AB157-04B9-4BAD-B580-C3C87C17828C}
|
||||
{1809EEF7-0772-404A-96C2-D76D80F1D191} = {4A7226CF-57BF-4CA3-A4AC-91A398A1D84B}
|
||||
{DD1E1B3F-4FA2-4F3A-9AE1-6B2A0B864AAF} = {4A7226CF-57BF-4CA3-A4AC-91A398A1D84B}
|
||||
EndGlobalSection
|
||||
GlobalSection(ExtensibilityGlobals) = postSolution
|
||||
SolutionGuid = {238A9613-5411-41CF-BDEC-168CCD5C03FB}
|
||||
EndGlobalSection
|
||||
EndGlobal
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
All contents of this distribution, unless explicitly marked otherwise,
|
||||
are subject to the following MIT license. Some files are marked to be under
|
||||
the Apache 2.0 license, which is included in the file LICENSE-APACHE.
|
||||
--------------------------------------------------------------------------------------------
|
||||
|
||||
MIT License
|
||||
|
||||
Copyright (c) .NET Foundation. All rights reserved.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE
|
||||
|
|
@ -0,0 +1,174 @@
|
|||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
# Introduction
|
||||
|
||||
DurableTask.Netherite is a backend storage provider for the Durable Task Framework (DTFx).
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
|
||||
<configuration>
|
||||
<packageSources>
|
||||
<!--<add key="localdurabletask" value="..\durabletask\build_output\packages" />
|
||||
<add key="localdurablefunctions-release" value="..\azure-functions-durable-extension\src\WebJobs.Extensions.DurableTask\bin\Release" />
|
||||
<add key="localdurablefunctions-debug" value="..\azure-functions-durable-extension\src\WebJobs.Extensions.DurableTask\bin\Debug" />-->
|
||||
<add key="nuget.org" value="https://www.nuget.org/api/v2/" />
|
||||
<add key="azure-appservice-staging" value="https://www.myget.org/F/azure-appservice-staging/api/v3/index.json" />
|
||||
<add key="azure-appservice" value="https://www.myget.org/F/azure-appservice/api/v3/index.json" />
|
||||
</packageSources>
|
||||
</configuration>
|
||||
Двоичный файл не отображается.
|
|
@ -0,0 +1,90 @@
|
|||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License. See LICENSE in the project root for license information.
|
||||
|
||||
namespace DurableTask.Netherite.AzureFunctions
|
||||
{
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Threading;
|
||||
using Microsoft.Azure.Storage;
|
||||
using Microsoft.Azure.Storage.Blob;
|
||||
|
||||
/// <summary>
|
||||
/// A simple utility class for writing text to an append blob in Azure Storage, using a periodic timer.
|
||||
/// For testing and debugging, not meant for production use.
|
||||
/// </summary>
|
||||
class BlobLogger
|
||||
{
|
||||
readonly DateTime starttime;
|
||||
readonly CloudAppendBlob blob;
|
||||
readonly object flushLock = new object();
|
||||
readonly object lineLock = new object();
|
||||
MemoryStream memoryStream;
|
||||
StreamWriter writer;
|
||||
|
||||
#pragma warning disable IDE0052 // Cannot remove timer reference, otherwise timer is garbage-collected and stops
|
||||
readonly Timer timer;
|
||||
#pragma warning restore IDE0052
|
||||
|
||||
public BlobLogger(string storageConnectionString, string workerId)
|
||||
{
|
||||
this.starttime = DateTime.UtcNow;
|
||||
|
||||
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageConnectionString);
|
||||
CloudBlobClient client = storageAccount.CreateCloudBlobClient();
|
||||
CloudBlobContainer container = client.GetContainerReference("logs");
|
||||
container.CreateIfNotExists();
|
||||
this.blob = container.GetAppendBlobReference($"{workerId}.{this.starttime:o}.log");
|
||||
this.blob.CreateOrReplace();
|
||||
|
||||
this.memoryStream = new MemoryStream();
|
||||
this.writer = new StreamWriter(this.memoryStream);
|
||||
|
||||
int interval = 14000 + new Random().Next(1000);
|
||||
this.timer = new Timer(this.Flush, null, interval, interval);
|
||||
}
|
||||
|
||||
public void WriteLine(string line)
|
||||
{
|
||||
lock (this.lineLock)
|
||||
{
|
||||
this.writer.WriteLine(line);
|
||||
}
|
||||
}
|
||||
|
||||
public void Flush(object ignored)
|
||||
{
|
||||
if (Monitor.TryEnter(this.flushLock))
|
||||
{
|
||||
try
|
||||
{
|
||||
MemoryStream toSave = null;
|
||||
|
||||
// grab current buffer and create new one
|
||||
lock (this.lineLock)
|
||||
{
|
||||
this.writer.Flush();
|
||||
if (this.memoryStream.Position > 0)
|
||||
{
|
||||
toSave = this.memoryStream;
|
||||
this.memoryStream = new MemoryStream();
|
||||
this.writer = new StreamWriter(this.memoryStream);
|
||||
}
|
||||
}
|
||||
|
||||
if (toSave != null)
|
||||
{
|
||||
// save to storage
|
||||
toSave.Seek(0, SeekOrigin.Begin);
|
||||
this.blob.AppendFromStream(toSave);
|
||||
toSave.Dispose();
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
Monitor.Exit(this.flushLock);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>netstandard2.1</TargetFramework>
|
||||
<LangVersion>8.0</LangVersion>
|
||||
<GeneratePackageOnBuild>true</GeneratePackageOnBuild>
|
||||
<IncludeSymbols>true</IncludeSymbols>
|
||||
<DebugType>embedded</DebugType>
|
||||
<PublishRepositoryUrl>true</PublishRepositoryUrl>
|
||||
<EmbedUntrackedSources>true</EmbedUntrackedSources>
|
||||
<SignAssembly>true</SignAssembly>
|
||||
<AssemblyOriginatorKeyFile>..\..\sign.snk</AssemblyOriginatorKeyFile>
|
||||
</PropertyGroup>
|
||||
|
||||
<!-- Version settings: https://andrewlock.net/version-vs-versionsuffix-vs-packageversion-what-do-they-all-mean/ -->
|
||||
<PropertyGroup>
|
||||
<MajorVersion>0</MajorVersion>
|
||||
<VersionPrefix>$(MajorVersion).1.0</VersionPrefix>
|
||||
<VersionSuffix>alpha</VersionSuffix>
|
||||
<AssemblyVersion>$(MajorVersion).0.0.0</AssemblyVersion>
|
||||
<BuildSuffix Condition="'$(GITHUB_RUN_NUMBER)' != ''">.$(GITHUB_RUN_NUMBER)</BuildSuffix>
|
||||
<FileVersion>$(VersionPrefix)$(BuildSuffix)</FileVersion>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Azure.Functions.Extensions" Version="1.0.0" />
|
||||
<PackageReference Include="Microsoft.Azure.WebJobs.Extensions.DurableTask" Version="2.3.1" />
|
||||
<PackageReference Include="Microsoft.Azure.DurableTask.Core" Version="2.4.1" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\DurableTask.Netherite\DurableTask.Netherite.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
|
@ -0,0 +1,94 @@
|
|||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License. See LICENSE in the project root for license information.
|
||||
|
||||
namespace DurableTask.Netherite.AzureFunctions
|
||||
{
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Threading;
|
||||
using Microsoft.Azure.Storage;
|
||||
using Microsoft.Azure.Storage.Blob;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
class LoggerFactoryWrapper : ILoggerFactory
|
||||
{
|
||||
readonly ILoggerFactory loggerFactory;
|
||||
readonly NetheriteProviderFactory providerFactory;
|
||||
readonly string hubName;
|
||||
readonly string workerId;
|
||||
|
||||
public LoggerFactoryWrapper(ILoggerFactory loggerFactory, string hubName, string workerId, NetheriteProviderFactory providerFactory)
|
||||
{
|
||||
this.hubName = hubName;
|
||||
this.workerId = workerId;
|
||||
this.loggerFactory = loggerFactory;
|
||||
this.providerFactory = providerFactory;
|
||||
}
|
||||
|
||||
public void AddProvider(ILoggerProvider provider)
|
||||
{
|
||||
this.loggerFactory.AddProvider(provider);
|
||||
}
|
||||
|
||||
public ILogger CreateLogger(string categoryName)
|
||||
{
|
||||
var logger = this.loggerFactory.CreateLogger(categoryName);
|
||||
return new LoggerWrapper(logger, categoryName, this.hubName, this.workerId, this.providerFactory);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
this.loggerFactory.Dispose();
|
||||
}
|
||||
|
||||
class LoggerWrapper : ILogger
|
||||
{
|
||||
readonly ILogger logger;
|
||||
readonly string prefix;
|
||||
readonly string hubName;
|
||||
readonly NetheriteProviderFactory providerFactory;
|
||||
readonly bool fullTracing;
|
||||
|
||||
public LoggerWrapper(ILogger logger, string category, string hubName, string workerId, NetheriteProviderFactory providerFactory)
|
||||
{
|
||||
this.logger = logger;
|
||||
this.prefix = $"{workerId} [{category}]";
|
||||
this.hubName = hubName;
|
||||
this.providerFactory = providerFactory;
|
||||
this.fullTracing = this.providerFactory.TraceToBlob || this.providerFactory.TraceToConsole;
|
||||
}
|
||||
|
||||
public IDisposable BeginScope<TState>(TState state)
|
||||
{
|
||||
return this.logger.BeginScope(state);
|
||||
}
|
||||
|
||||
public bool IsEnabled(Microsoft.Extensions.Logging.LogLevel logLevel)
|
||||
{
|
||||
return this.fullTracing || this.logger.IsEnabled(logLevel);
|
||||
}
|
||||
|
||||
public void Log<TState>(Microsoft.Extensions.Logging.LogLevel logLevel, Microsoft.Extensions.Logging.EventId eventId, TState state, Exception exception, Func<TState, Exception, string> formatter)
|
||||
{
|
||||
if (this.IsEnabled(logLevel))
|
||||
{
|
||||
this.logger.Log(logLevel, eventId, state, exception, formatter);
|
||||
|
||||
if (this.providerFactory.TraceToConsole || this.providerFactory.TraceToBlob)
|
||||
{
|
||||
string formattedString = $"{DateTime.UtcNow:o} {this.prefix}s{(int)logLevel} {formatter(state, exception)}";
|
||||
|
||||
if (this.providerFactory.TraceToConsole)
|
||||
{
|
||||
System.Console.WriteLine(formattedString);
|
||||
}
|
||||
|
||||
NetheriteProviderFactory.BlobLogger?.WriteLine(formattedString);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License. See LICENSE in the project root for license information.
|
||||
|
||||
namespace DurableTask.Netherite.AzureFunctions
|
||||
{
|
||||
using System;
|
||||
using DurableTask.Netherite;
|
||||
using Newtonsoft.Json;
|
||||
|
||||
public class NetheriteOptions
|
||||
{
|
||||
[JsonProperty("connectionStringName")]
|
||||
public string ConnectionStringName { get; set; } = "UseDevelopmentStorage=true;";
|
||||
|
||||
[JsonProperty("taskEventLockTimeout")]
|
||||
public TimeSpan TaskEventLockTimeout { get; set; } = TimeSpan.FromMinutes(2);
|
||||
|
||||
internal NetheriteOrchestrationServiceSettings ProviderOptions { get; set; } = new NetheriteOrchestrationServiceSettings();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,106 @@
|
|||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License. See LICENSE in the project root for license information.
|
||||
|
||||
namespace DurableTask.Netherite.AzureFunctions
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using DurableTask.Core;
|
||||
using DurableTask.Netherite;
|
||||
using Microsoft.Azure.WebJobs.Extensions.DurableTask;
|
||||
using Newtonsoft.Json;
|
||||
using Newtonsoft.Json.Linq;
|
||||
|
||||
class NetheriteProvider : DurabilityProvider
|
||||
{
|
||||
public NetheriteOrchestrationService Service { get; }
|
||||
public NetheriteOrchestrationServiceSettings Settings { get; }
|
||||
|
||||
public NetheriteProvider(
|
||||
NetheriteOrchestrationService service,
|
||||
NetheriteOrchestrationServiceSettings settings)
|
||||
: base("Netherite", service, service, settings.StorageConnectionString)
|
||||
{
|
||||
this.Service = service;
|
||||
this.Settings = settings;
|
||||
}
|
||||
|
||||
public override JObject ConfigurationJson => JObject.FromObject(this.Settings);
|
||||
|
||||
public override bool SupportsEntities => true;
|
||||
|
||||
public override bool SupportsPollFreeWait => true;
|
||||
|
||||
public override bool GuaranteesOrderedDelivery => true;
|
||||
|
||||
public override TimeSpan MaximumDelayTime { get; set; } = TimeSpan.MaxValue;
|
||||
|
||||
/// <summary>
|
||||
/// The app setting containing the Azure Storage connection string.
|
||||
/// </summary>
|
||||
public override string ConnectionName => "StorageConnectionString";
|
||||
|
||||
/// <inheritdoc/>
|
||||
public async override Task<string> RetrieveSerializedEntityState(EntityId entityId, JsonSerializerSettings serializerSettings)
|
||||
{
|
||||
var instanceId = this.GetSchedulerIdFromEntityId(entityId);
|
||||
OrchestrationState state = await this.Service.GetOrchestrationStateAsync(instanceId, true, true);
|
||||
if (this.TryGetEntityStateFromSerializedSchedulerState(state, serializerSettings, out string result))
|
||||
{
|
||||
return result;
|
||||
}
|
||||
else
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public async override Task<IList<OrchestrationState>> GetOrchestrationStateWithInputsAsync(string instanceId, bool showInput = true)
|
||||
{
|
||||
var result = new List<OrchestrationState>();
|
||||
var state = await this.Service.GetOrchestrationStateAsync(instanceId, showInput, true);
|
||||
if (state != null)
|
||||
{
|
||||
result.Add(state);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public async override Task<PurgeHistoryResult> PurgeInstanceHistoryByInstanceId(string instanceId)
|
||||
{
|
||||
var numberInstancesDeleted = await this.Service.PurgeInstanceHistoryAsync(instanceId);
|
||||
return new PurgeHistoryResult(numberInstancesDeleted);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public override Task<int> PurgeHistoryByFilters(DateTime createdTimeFrom, DateTime? createdTimeTo, IEnumerable<OrchestrationStatus> runtimeStatus)
|
||||
{
|
||||
return this.Service.PurgeInstanceHistoryAsync(createdTimeFrom, createdTimeTo, runtimeStatus);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public async override Task<OrchestrationStatusQueryResult> GetOrchestrationStateWithPagination(OrchestrationStatusQueryCondition condition, CancellationToken cancellationToken)
|
||||
{
|
||||
var instanceQuery = new InstanceQuery(
|
||||
runtimeStatus: condition.RuntimeStatus?.Select(p => (OrchestrationStatus)Enum.Parse(typeof(OrchestrationStatus), p.ToString())).ToArray(),
|
||||
createdTimeFrom: (condition.CreatedTimeFrom == default) ? (DateTime?)null : condition.CreatedTimeFrom.ToUniversalTime(),
|
||||
createdTimeTo: (condition.CreatedTimeTo == default) ? (DateTime?)null : condition.CreatedTimeTo.ToUniversalTime(),
|
||||
instanceIdPrefix: condition.InstanceIdPrefix,
|
||||
fetchInput: condition.ShowInput);
|
||||
|
||||
InstanceQueryResult result = await this.Service.QueryOrchestrationStatesAsync(instanceQuery, condition.PageSize, condition.ContinuationToken, cancellationToken);
|
||||
|
||||
return new OrchestrationStatusQueryResult()
|
||||
{
|
||||
DurableOrchestrationState = result.Instances.Select(ostate => this.ConvertOrchestrationStateToStatus(ostate)).ToList(),
|
||||
ContinuationToken = result.ContinuationToken,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,192 @@
|
|||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License. See LICENSE in the project root for license information.
|
||||
|
||||
namespace DurableTask.Netherite.AzureFunctions
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using DurableTask.Core;
|
||||
using DurableTask.Netherite;
|
||||
using Microsoft.Azure.WebJobs.Extensions.DurableTask;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Newtonsoft.Json;
|
||||
|
||||
public class NetheriteProviderFactory : IDurabilityProviderFactory
|
||||
{
|
||||
readonly ConcurrentDictionary<DurableClientAttribute, NetheriteProvider> cachedProviders
|
||||
= new ConcurrentDictionary<DurableClientAttribute, NetheriteProvider>();
|
||||
|
||||
readonly DurableTaskOptions extensionOptions;
|
||||
readonly IConnectionStringResolver connectionStringResolver;
|
||||
|
||||
// the following are boolean options that can be specified in host.json,
|
||||
// but are not passed on to the backend
|
||||
public bool TraceToConsole { get; }
|
||||
public bool TraceToBlob { get; }
|
||||
|
||||
NetheriteProvider defaultProvider;
|
||||
ILoggerFactory loggerFactory;
|
||||
|
||||
internal static BlobLogger BlobLogger { get; set; }
|
||||
|
||||
// Called by the Azure Functions runtime dependency injection infrastructure
|
||||
public NetheriteProviderFactory(
|
||||
IOptions<DurableTaskOptions> extensionOptions,
|
||||
ILoggerFactory loggerFactory,
|
||||
IConnectionStringResolver connectionStringResolver)
|
||||
{
|
||||
this.extensionOptions = extensionOptions?.Value ?? throw new ArgumentNullException(nameof(extensionOptions));
|
||||
this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));
|
||||
this.connectionStringResolver = connectionStringResolver ?? throw new ArgumentNullException(nameof(connectionStringResolver));
|
||||
|
||||
bool ReadBooleanSetting(string name) => this.extensionOptions.StorageProvider.TryGetValue(name, out object objValue)
|
||||
&& objValue is string stringValue && bool.TryParse(stringValue, out bool boolValue) && boolValue;
|
||||
|
||||
this.TraceToConsole = ReadBooleanSetting(nameof(this.TraceToConsole));
|
||||
this.TraceToBlob = ReadBooleanSetting(nameof(this.TraceToBlob));
|
||||
}
|
||||
|
||||
NetheriteOrchestrationServiceSettings GetNetheriteOrchestrationServiceSettings(string taskHubNameOverride = null)
|
||||
{
|
||||
var eventSourcedSettings = new NetheriteOrchestrationServiceSettings();
|
||||
|
||||
// override DTFx defaults to the defaults we want to use in DF
|
||||
eventSourcedSettings.ThrowExceptionOnInvalidDedupeStatus = true;
|
||||
|
||||
// copy all applicable fields from both the options and the storageProvider options
|
||||
JsonConvert.PopulateObject(JsonConvert.SerializeObject(this.extensionOptions), eventSourcedSettings);
|
||||
JsonConvert.PopulateObject(JsonConvert.SerializeObject(this.extensionOptions.StorageProvider), eventSourcedSettings);
|
||||
|
||||
// resolve any indirection in the specification of the two connection strings
|
||||
eventSourcedSettings.StorageConnectionString = this.ResolveIndirection(
|
||||
eventSourcedSettings.StorageConnectionString,
|
||||
nameof(NetheriteOrchestrationServiceSettings.StorageConnectionString));
|
||||
eventSourcedSettings.EventHubsConnectionString = this.ResolveIndirection(
|
||||
eventSourcedSettings.EventHubsConnectionString,
|
||||
nameof(NetheriteOrchestrationServiceSettings.EventHubsConnectionString));
|
||||
|
||||
// if worker id is specified in environment, it overrides the configured setting
|
||||
string workerId = Environment.GetEnvironmentVariable("WorkerId");
|
||||
if (!string.IsNullOrEmpty(workerId))
|
||||
{
|
||||
eventSourcedSettings.WorkerId = workerId;
|
||||
}
|
||||
|
||||
eventSourcedSettings.HubName = this.extensionOptions.HubName;
|
||||
|
||||
if (taskHubNameOverride != null)
|
||||
{
|
||||
eventSourcedSettings.HubName = taskHubNameOverride;
|
||||
}
|
||||
|
||||
// TODO sanitize hubname
|
||||
|
||||
return eventSourcedSettings;
|
||||
}
|
||||
|
||||
public void CreateDefaultProvider()
|
||||
{
|
||||
var settings = this.GetNetheriteOrchestrationServiceSettings();
|
||||
|
||||
if (this.TraceToBlob && BlobLogger == null)
|
||||
{
|
||||
BlobLogger = new BlobLogger(settings.StorageConnectionString, settings.WorkerId);
|
||||
}
|
||||
|
||||
if (this.TraceToConsole || this.TraceToBlob)
|
||||
{
|
||||
// capture trace events generated in the backend and redirect them to additional sinks
|
||||
this.loggerFactory = new LoggerFactoryWrapper(this.loggerFactory, settings.HubName, settings.WorkerId, this);
|
||||
}
|
||||
|
||||
var key = new DurableClientAttribute()
|
||||
{
|
||||
TaskHub = settings.HubName,
|
||||
ConnectionName = settings.StorageConnectionString,
|
||||
};
|
||||
|
||||
var service = new NetheriteOrchestrationService(settings, this.loggerFactory);
|
||||
this.defaultProvider = new NetheriteProvider(service, settings);
|
||||
this.cachedProviders[key] = this.defaultProvider;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public DurabilityProvider GetDurabilityProvider()
|
||||
{
|
||||
if (this.defaultProvider == null)
|
||||
{
|
||||
this.CreateDefaultProvider();
|
||||
}
|
||||
|
||||
return this.defaultProvider;
|
||||
}
|
||||
|
||||
// Called by the Durable client binding infrastructure
|
||||
public DurabilityProvider GetDurabilityProvider(DurableClientAttribute attribute)
|
||||
{
|
||||
var settings = this.GetNetheriteOrchestrationServiceSettings(attribute.TaskHub);
|
||||
|
||||
if (string.Equals(this.defaultProvider.Settings.HubName, settings.HubName, StringComparison.OrdinalIgnoreCase) &&
|
||||
string.Equals(this.defaultProvider.Settings.StorageConnectionString, settings.StorageConnectionString, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return this.defaultProvider;
|
||||
}
|
||||
|
||||
DurableClientAttribute key = new DurableClientAttribute()
|
||||
{
|
||||
TaskHub = settings.HubName,
|
||||
ConnectionName = settings.StorageConnectionString,
|
||||
};
|
||||
|
||||
return this.cachedProviders.GetOrAdd(key, _ =>
|
||||
{
|
||||
//TODO support client-only version
|
||||
var service = new NetheriteOrchestrationService(settings, this.loggerFactory);
|
||||
return new NetheriteProvider(service, settings);
|
||||
});
|
||||
}
|
||||
|
||||
string ResolveIndirection(string value, string propertyName)
|
||||
{
|
||||
string envName;
|
||||
string setting;
|
||||
|
||||
if (string.IsNullOrEmpty(value))
|
||||
{
|
||||
envName = propertyName;
|
||||
}
|
||||
else if (value.StartsWith("$"))
|
||||
{
|
||||
envName = value.Substring(1);
|
||||
}
|
||||
else if (value.StartsWith("%") && value.EndsWith("%"))
|
||||
{
|
||||
envName = value.Substring(1, value.Length - 2);
|
||||
}
|
||||
else
|
||||
{
|
||||
envName = null;
|
||||
}
|
||||
|
||||
if (envName != null)
|
||||
{
|
||||
setting = this.connectionStringResolver.Resolve(envName);
|
||||
}
|
||||
else
|
||||
{
|
||||
setting = value;
|
||||
}
|
||||
|
||||
if (string.IsNullOrEmpty(setting))
|
||||
{
|
||||
throw new InvalidOperationException($"Could not resolve '{envName}' for required property '{propertyName}' in EventSourced storage provider settings.");
|
||||
}
|
||||
else
|
||||
{
|
||||
return setting;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
// Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the MIT License. See LICENSE in the project root for license information.
|
||||
|
||||
// Reference: https://docs.microsoft.com/en-us/azure/azure-functions/functions-dotnet-dependency-injection
|
||||
[assembly: Microsoft.Azure.Functions.Extensions.DependencyInjection.FunctionsStartup(
|
||||
typeof(DurableTask.Netherite.AzureFunctions.NetheriteProviderStartup))]
|
||||
|
||||
namespace DurableTask.Netherite.AzureFunctions
|
||||
{
|
||||
using System.Collections.Concurrent;
|
||||
using Microsoft.Azure.Functions.Extensions.DependencyInjection;
|
||||
using Microsoft.Azure.WebJobs.Extensions.DurableTask;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
|
||||
class NetheriteProviderStartup : FunctionsStartup
|
||||
{
|
||||
public override void Configure(IFunctionsHostBuilder builder)
|
||||
{
|
||||
builder.Services.AddSingleton<IDurabilityProviderFactory, NetheriteProviderFactory>();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System.Threading.Tasks;
|
||||
|
||||
/// <summary>
|
||||
/// The functionality for storing and recovering partition states.
|
||||
/// </summary>
|
||||
interface IStorageProvider
|
||||
{
|
||||
/// <summary>
|
||||
/// Creates a <see cref="IPartitionState"/> object that represents the partition state.
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
IPartitionState CreatePartitionState();
|
||||
|
||||
/// <summary>
|
||||
/// Deletes all partition states.
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
Task DeleteAllPartitionStatesAsync();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System.Threading.Tasks;
|
||||
|
||||
/// <summary>
|
||||
/// Top-level functionality for starting and stopping the transport back-end on a machine.
|
||||
/// </summary>
|
||||
public interface ITaskHub
|
||||
{
|
||||
/// <summary>
|
||||
/// Tests whether this taskhub exists in storage.
|
||||
/// </summary>
|
||||
/// <returns>true if this taskhub has been created in storage.</returns>
|
||||
Task<bool> ExistsAsync();
|
||||
|
||||
/// <summary>
|
||||
/// Creates this taskhub in storage.
|
||||
/// </summary>
|
||||
/// <returns>after the taskhub has been created in storage.</returns>
|
||||
Task CreateAsync();
|
||||
|
||||
/// <summary>
|
||||
/// Deletes this taskhub and all of its associated data in storage.
|
||||
/// </summary>
|
||||
/// <returns>after the taskhub has been deleted from storage.</returns>
|
||||
Task DeleteAsync();
|
||||
|
||||
/// <summary>
|
||||
/// Starts the transport backend. Creates a client and registers with the transport provider so partitions may be placed on this host.
|
||||
/// </summary>
|
||||
/// <returns>After the transport backend has started and created the client.</returns>
|
||||
Task StartAsync();
|
||||
|
||||
/// <summary>
|
||||
/// Stops the transport backend.
|
||||
/// </summary>
|
||||
/// <param name="isForced">Whether to shut down as quickly as possible, or gracefully.</param>
|
||||
/// <returns>After the transport backend has stopped.</returns>
|
||||
Task StopAsync(bool isForced);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,226 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading.Tasks;
|
||||
using DurableTask.Core;
|
||||
using DurableTask.Core.Common;
|
||||
|
||||
/// <summary>
|
||||
/// Is used while applying an effect to a partition state, to carry
|
||||
/// information about the context, and to enumerate the objects on which the effect
|
||||
/// is being processed.
|
||||
/// </summary>
|
||||
class EffectTracker : List<TrackedObjectKey>
|
||||
{
|
||||
readonly Func<TrackedObjectKey, EffectTracker, ValueTask> applyToStore;
|
||||
readonly Func<(long, long)> getPositions;
|
||||
readonly System.Diagnostics.Stopwatch stopWatch;
|
||||
|
||||
public EffectTracker(Partition partition, Func<TrackedObjectKey, EffectTracker, ValueTask> applyToStore, Func<(long, long)> getPositions)
|
||||
{
|
||||
this.Partition = partition;
|
||||
this.applyToStore = applyToStore;
|
||||
this.getPositions = getPositions;
|
||||
this.stopWatch = new System.Diagnostics.Stopwatch();
|
||||
this.stopWatch.Start();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The current partition.
|
||||
/// </summary>
|
||||
public Partition Partition { get; }
|
||||
|
||||
/// <summary>
|
||||
/// The effect currently being applied.
|
||||
/// </summary>
|
||||
public dynamic Effect { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// True if we are replaying this effect during recovery.
|
||||
/// Typically, external side effects (such as launching tasks, sending responses, etc.)
|
||||
/// are suppressed during replay.
|
||||
/// </summary>
|
||||
public bool IsReplaying { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Applies the event to the given tracked object, using dynamic dispatch to
|
||||
/// select the correct Process method overload for the event.
|
||||
/// </summary>
|
||||
/// <param name="trackedObject"></param>
|
||||
/// <remarks>Called by the storage layer when this object calls applyToStore.</remarks>
|
||||
public void ProcessEffectOn(dynamic trackedObject)
|
||||
{
|
||||
trackedObject.Process(this.Effect, this);
|
||||
}
|
||||
|
||||
public async Task ProcessUpdate(PartitionUpdateEvent updateEvent)
|
||||
{
|
||||
(long commitLogPosition, long inputQueuePosition) = this.getPositions();
|
||||
double startedTimestamp = this.Partition.CurrentTimeMs;
|
||||
|
||||
using (EventTraceContext.MakeContext(commitLogPosition, updateEvent.EventIdString))
|
||||
{
|
||||
try
|
||||
{
|
||||
this.Partition.EventDetailTracer?.TraceEventProcessingStarted(commitLogPosition, updateEvent, this.IsReplaying);
|
||||
|
||||
this.Effect = updateEvent;
|
||||
|
||||
// collect the initial list of targets
|
||||
updateEvent.DetermineEffects(this);
|
||||
|
||||
// process until there are no more targets
|
||||
while (this.Count > 0)
|
||||
{
|
||||
await ProcessRecursively().ConfigureAwait(false);
|
||||
}
|
||||
|
||||
async ValueTask ProcessRecursively()
|
||||
{
|
||||
var startPos = this.Count - 1;
|
||||
var key = this[startPos];
|
||||
|
||||
this.Partition.EventDetailTracer?.TraceEventProcessingDetail($"Process on [{key}]");
|
||||
|
||||
// start with processing the event on this object
|
||||
await this.applyToStore(key, this).ConfigureAwait(false);
|
||||
|
||||
// recursively process all additional objects to process
|
||||
while (this.Count - 1 > startPos)
|
||||
{
|
||||
await ProcessRecursively().ConfigureAwait(false);
|
||||
}
|
||||
|
||||
// pop this object now since we are done processing
|
||||
this.RemoveAt(startPos);
|
||||
}
|
||||
|
||||
this.Effect = null;
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// o.k. during termination
|
||||
}
|
||||
catch (Exception exception) when (!Utils.IsFatal(exception))
|
||||
{
|
||||
// for robustness, swallow exceptions, but report them
|
||||
this.Partition.ErrorHandler.HandleError(nameof(ProcessUpdate), $"Encountered exception while processing update event {updateEvent}", exception, false, false);
|
||||
}
|
||||
finally
|
||||
{
|
||||
double finishedTimestamp = this.Partition.CurrentTimeMs;
|
||||
this.Partition.EventTraceHelper.TraceEventProcessed(commitLogPosition, updateEvent, startedTimestamp, finishedTimestamp, this.IsReplaying);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void ProcessReadResult(PartitionReadEvent readEvent, TrackedObjectKey key, TrackedObject target)
|
||||
{
|
||||
if (readEvent == null)
|
||||
{
|
||||
// this read is not caused by a read event but was issued directly
|
||||
// in that case we are not processing the result here
|
||||
return;
|
||||
}
|
||||
|
||||
(long commitLogPosition, long inputQueuePosition) = this.getPositions();
|
||||
this.Partition.Assert(!this.IsReplaying); // read events are never part of the replay
|
||||
double startedTimestamp = this.Partition.CurrentTimeMs;
|
||||
|
||||
using (EventTraceContext.MakeContext(commitLogPosition, readEvent.EventIdString))
|
||||
{
|
||||
try
|
||||
{
|
||||
readEvent.Deliver(key, target, out var isReady);
|
||||
|
||||
if (isReady)
|
||||
{
|
||||
this.Partition.EventDetailTracer?.TraceEventProcessingStarted(commitLogPosition, readEvent, false);
|
||||
|
||||
// trace read accesses to instance and history
|
||||
switch (key.ObjectType)
|
||||
{
|
||||
case TrackedObjectKey.TrackedObjectType.Instance:
|
||||
InstanceState instanceState = (InstanceState)target;
|
||||
string instanceExecutionId = instanceState?.OrchestrationState?.OrchestrationInstance.ExecutionId;
|
||||
string status = instanceState?.OrchestrationState?.OrchestrationStatus.ToString() ?? "null";
|
||||
this.Partition.EventTraceHelper.TraceFetchedInstanceStatus(readEvent, key.InstanceId, instanceExecutionId, status, startedTimestamp - readEvent.IssuedTimestamp);
|
||||
break;
|
||||
|
||||
case TrackedObjectKey.TrackedObjectType.History:
|
||||
HistoryState historyState = (HistoryState)target;
|
||||
string historyExecutionId = historyState?.ExecutionId;
|
||||
int eventCount = historyState?.History?.Count ?? 0;
|
||||
int episode = historyState?.Episode ?? 0;
|
||||
this.Partition.EventTraceHelper.TraceFetchedInstanceHistory(readEvent, key.InstanceId, historyExecutionId, eventCount, episode, startedTimestamp - readEvent.IssuedTimestamp);
|
||||
break;
|
||||
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
readEvent.Fire(this.Partition);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// o.k. during termination
|
||||
}
|
||||
catch (Exception exception) when (!Utils.IsFatal(exception))
|
||||
{
|
||||
// for robustness, swallow exceptions, but report them
|
||||
this.Partition.ErrorHandler.HandleError(nameof(ProcessReadResult), $"Encountered exception while processing read event {readEvent}", exception, false, false);
|
||||
}
|
||||
finally
|
||||
{
|
||||
double finishedTimestamp = this.Partition.CurrentTimeMs;
|
||||
this.Partition.EventTraceHelper.TraceEventProcessed(commitLogPosition, readEvent, startedTimestamp, finishedTimestamp, false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public async Task ProcessQueryResultAsync(PartitionQueryEvent queryEvent, IAsyncEnumerable<OrchestrationState> instances)
|
||||
{
|
||||
(long commitLogPosition, long inputQueuePosition) = this.getPositions();
|
||||
this.Partition.Assert(!this.IsReplaying); // query events are never part of the replay
|
||||
double startedTimestamp = this.Partition.CurrentTimeMs;
|
||||
|
||||
using (EventTraceContext.MakeContext(commitLogPosition, queryEvent.EventIdString))
|
||||
{
|
||||
try
|
||||
{
|
||||
this.Partition.EventDetailTracer?.TraceEventProcessingStarted(commitLogPosition, queryEvent, false);
|
||||
await queryEvent.OnQueryCompleteAsync(instances, this.Partition);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// o.k. during termination
|
||||
}
|
||||
catch (Exception exception) when (!Utils.IsFatal(exception))
|
||||
{
|
||||
// for robustness, swallow exceptions, but report them
|
||||
this.Partition.ErrorHandler.HandleError(nameof(ProcessQueryResultAsync), $"Encountered exception while processing query event {queryEvent}", exception, false, false);
|
||||
}
|
||||
finally
|
||||
{
|
||||
double finishedTimestamp = this.Partition.CurrentTimeMs;
|
||||
this.Partition.EventTraceHelper.TraceEventProcessed(commitLogPosition, queryEvent, startedTimestamp, finishedTimestamp, false);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
|
||||
/// <summary>
|
||||
/// A handler for fatal or non-fatal errors encountered in a partition.
|
||||
/// </summary>
|
||||
public interface IPartitionErrorHandler
|
||||
{
|
||||
/// <summary>
|
||||
/// A cancellation token that is cancelled when the partition is terminated.
|
||||
/// </summary>
|
||||
CancellationToken Token { get; }
|
||||
|
||||
/// <summary>
|
||||
/// A boolean indicating whether the partition is terminated.
|
||||
/// </summary>
|
||||
bool IsTerminated { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Error handling for the partition.
|
||||
/// </summary>
|
||||
/// <param name="where">A brief description of the component that observed the error.</param>
|
||||
/// <param name="message">A message describing the circumstances.</param>
|
||||
/// <param name="e">The exception that was observed, or null.</param>
|
||||
/// <param name="terminatePartition">whether this partition should be terminated (i.e. recycle and recover from storage).</param>
|
||||
/// <param name="reportAsWarning">whether this error should be reported with the severity of a warning.</param>
|
||||
void HandleError(string where, string message, Exception e, bool terminatePartition, bool reportAsWarning);
|
||||
|
||||
/// <summary>
|
||||
/// Terminates the partition normally, after shutdown.
|
||||
/// </summary>
|
||||
void TerminateNormally();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
/// <summary>
|
||||
/// Abstraction for a storage-backed, event-sourced partition.
|
||||
/// </summary>
|
||||
interface IPartitionState
|
||||
{
|
||||
/// <summary>
|
||||
/// Restore the state of a partition from storage, or create a new one if there is nothing stored.
|
||||
/// </summary>
|
||||
/// <param name="localPartition">The partition.</param>
|
||||
/// <param name="errorHandler">An error handler to initiate and/or indicate termination of this partition.</param>
|
||||
/// <param name="firstInputQueuePosition">For new partitions, the position of the first message to receive.</param>
|
||||
/// <returns>the input queue position from which to resume input processing</returns>
|
||||
/// <exception cref="OperationCanceledException">Indicates that termination was signaled before the operation completed.</exception>
|
||||
Task<long> CreateOrRestoreAsync(Partition localPartition, IPartitionErrorHandler errorHandler, long firstInputQueuePosition);
|
||||
|
||||
/// <summary>
|
||||
/// Finish processing events and save the partition state to storage.
|
||||
/// </summary>
|
||||
/// <param name="takeFinalCheckpoint">Whether to take a final state checkpoint.</param>
|
||||
/// <returns>A task that completes when the state has been saved.</returns>
|
||||
/// <exception cref="OperationCanceledException">Indicates that termination was signaled before the operation completed.</exception>
|
||||
Task CleanShutdown(bool takeFinalCheckpoint);
|
||||
|
||||
/// <summary>
|
||||
/// Queues an internal event (originating on this same partition)
|
||||
/// for processing on this partition.
|
||||
/// </summary>
|
||||
/// <param name="evt">The event to process.</param>
|
||||
void SubmitInternalEvent(PartitionEvent evt);
|
||||
|
||||
/// <summary>
|
||||
/// Queues external events (originating on clients or other partitions)
|
||||
/// for processing on this partition.
|
||||
/// </summary>
|
||||
/// <param name="evt">The collection of events to process.</param>
|
||||
void SubmitExternalEvents(IList<PartitionEvent> evt);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,98 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using DurableTask.Netherite.Scaling;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.Serialization;
|
||||
|
||||
/// <summary>
|
||||
/// An object whose value is persisted by storage, and that is indexed by a primary key.
|
||||
/// </summary>
|
||||
[DataContract]
|
||||
[KnownTypeAttribute("KnownTypes")]
|
||||
abstract class TrackedObject
|
||||
{
|
||||
/// <summary>
|
||||
/// The partition to which this object belongs.
|
||||
/// </summary>
|
||||
[IgnoreDataMember]
|
||||
public Partition Partition;
|
||||
|
||||
/// <summary>
|
||||
/// The key for this object.
|
||||
/// </summary>
|
||||
[IgnoreDataMember]
|
||||
public abstract TrackedObjectKey Key { get; }
|
||||
|
||||
/// <summary>
|
||||
/// The current value in serialized form, or null
|
||||
/// </summary>
|
||||
[IgnoreDataMember]
|
||||
internal byte[] SerializationCache { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The collection of all types of tracked objects and polymorphic members of tracked objects. Can be
|
||||
/// used by serializers to compute a type map.
|
||||
/// </summary>
|
||||
/// <returns>The collection of types.</returns>
|
||||
static IEnumerable<Type> KnownTypes()
|
||||
{
|
||||
foreach (var t in Core.History.HistoryEvent.KnownTypes())
|
||||
{
|
||||
yield return t;
|
||||
}
|
||||
foreach (var t in DurableTask.Netherite.Event.KnownTypes())
|
||||
{
|
||||
yield return t;
|
||||
}
|
||||
foreach (var t in TrackedObjectKey.TypeMap.Values)
|
||||
{
|
||||
yield return t;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Is called on all singleton objects once at the very beginning
|
||||
/// </summary>
|
||||
public virtual void OnFirstInitialization()
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Is automatically called on all singleton objects after recovery. Typically used to
|
||||
/// restart pending activities, timers, tasks and the like.
|
||||
/// </summary>
|
||||
public virtual void OnRecoveryCompleted()
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Is called to update the load information that is published
|
||||
/// </summary>
|
||||
/// <param name="info"></param>
|
||||
public virtual void UpdateLoadInfo(PartitionLoadInfo info)
|
||||
{
|
||||
}
|
||||
|
||||
public virtual void Process(PartitionEventFragment e, EffectTracker effects)
|
||||
{
|
||||
// processing a reassembled event just applies the original event
|
||||
dynamic dynamicThis = this;
|
||||
dynamic dynamicPartitionEvent = e.ReassembledEvent;
|
||||
dynamicThis.Process(dynamicPartitionEvent, effects);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,153 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
|
||||
/// <summary>
|
||||
/// Represents a key used to identify <see cref="TrackedObject"/> instances.
|
||||
/// </summary>
|
||||
struct TrackedObjectKey
|
||||
{
|
||||
public TrackedObjectType ObjectType;
|
||||
public string InstanceId;
|
||||
|
||||
public TrackedObjectKey(TrackedObjectType objectType) { this.ObjectType = objectType; this.InstanceId = null; }
|
||||
public TrackedObjectKey(TrackedObjectType objectType, string instanceId) { this.ObjectType = objectType; this.InstanceId = instanceId; }
|
||||
|
||||
public enum TrackedObjectType
|
||||
{
|
||||
// singletons
|
||||
Activities,
|
||||
Dedup,
|
||||
Outbox,
|
||||
Reassembly,
|
||||
Sessions,
|
||||
Timers,
|
||||
Prefetch,
|
||||
Queries,
|
||||
|
||||
// non-singletons
|
||||
History,
|
||||
Instance,
|
||||
}
|
||||
|
||||
public static Dictionary<TrackedObjectType, Type> TypeMap = new Dictionary<TrackedObjectType, Type>()
|
||||
{
|
||||
{ TrackedObjectType.Activities, typeof(ActivitiesState) },
|
||||
{ TrackedObjectType.Dedup, typeof(DedupState) },
|
||||
{ TrackedObjectType.Outbox, typeof(OutboxState) },
|
||||
{ TrackedObjectType.Reassembly, typeof(ReassemblyState) },
|
||||
{ TrackedObjectType.Sessions, typeof(SessionsState) },
|
||||
{ TrackedObjectType.Timers, typeof(TimersState) },
|
||||
{ TrackedObjectType.Prefetch, typeof(PrefetchState) },
|
||||
{ TrackedObjectType.Queries, typeof(QueriesState) },
|
||||
|
||||
// non-singletons
|
||||
{ TrackedObjectType.History, typeof(HistoryState) },
|
||||
{ TrackedObjectType.Instance, typeof(InstanceState) },
|
||||
};
|
||||
|
||||
public static bool IsSingletonType(TrackedObjectType t) => (int) t < (int) TrackedObjectType.History;
|
||||
|
||||
public bool IsSingleton => IsSingletonType(this.ObjectType);
|
||||
|
||||
public static int Compare(ref TrackedObjectKey key1, ref TrackedObjectKey key2)
|
||||
{
|
||||
int result = key1.ObjectType.CompareTo(key2.ObjectType);
|
||||
return result == 0 ? key1.InstanceId.CompareTo(key2.InstanceId) : result;
|
||||
}
|
||||
|
||||
public class Comparer : IComparer<TrackedObjectKey>
|
||||
{
|
||||
public int Compare(TrackedObjectKey x, TrackedObjectKey y) => TrackedObjectKey.Compare(ref x, ref y);
|
||||
}
|
||||
|
||||
public override int GetHashCode()
|
||||
{
|
||||
return (this.InstanceId?.GetHashCode() ?? 0) + (int) this.ObjectType;
|
||||
}
|
||||
|
||||
public override bool Equals(object obj)
|
||||
{
|
||||
return (obj is TrackedObjectKey other && this.ObjectType == other.ObjectType && this.InstanceId == other.InstanceId);
|
||||
}
|
||||
|
||||
// convenient constructors for singletons
|
||||
|
||||
public static TrackedObjectKey Activities = new TrackedObjectKey() { ObjectType = TrackedObjectType.Activities };
|
||||
public static TrackedObjectKey Dedup = new TrackedObjectKey() { ObjectType = TrackedObjectType.Dedup };
|
||||
public static TrackedObjectKey Outbox = new TrackedObjectKey() { ObjectType = TrackedObjectType.Outbox };
|
||||
public static TrackedObjectKey Reassembly = new TrackedObjectKey() { ObjectType = TrackedObjectType.Reassembly };
|
||||
public static TrackedObjectKey Sessions = new TrackedObjectKey() { ObjectType = TrackedObjectType.Sessions };
|
||||
public static TrackedObjectKey Timers = new TrackedObjectKey() { ObjectType = TrackedObjectType.Timers };
|
||||
public static TrackedObjectKey Prefetch = new TrackedObjectKey() { ObjectType = TrackedObjectType.Prefetch };
|
||||
public static TrackedObjectKey Queries = new TrackedObjectKey() { ObjectType = TrackedObjectType.Queries };
|
||||
|
||||
// convenient constructors for non-singletons
|
||||
|
||||
public static TrackedObjectKey History(string id) => new TrackedObjectKey()
|
||||
{
|
||||
ObjectType = TrackedObjectType.History,
|
||||
InstanceId = id,
|
||||
};
|
||||
public static TrackedObjectKey Instance(string id) => new TrackedObjectKey()
|
||||
{
|
||||
ObjectType = TrackedObjectType.Instance,
|
||||
InstanceId = id,
|
||||
};
|
||||
|
||||
public static TrackedObject Factory(TrackedObjectKey key) => key.ObjectType switch
|
||||
{
|
||||
TrackedObjectType.Activities => new ActivitiesState(),
|
||||
TrackedObjectType.Dedup => new DedupState(),
|
||||
TrackedObjectType.Outbox => new OutboxState(),
|
||||
TrackedObjectType.Reassembly => new ReassemblyState(),
|
||||
TrackedObjectType.Sessions => new SessionsState(),
|
||||
TrackedObjectType.Timers => new TimersState(),
|
||||
TrackedObjectType.Prefetch => new PrefetchState(),
|
||||
TrackedObjectType.Queries => new QueriesState(),
|
||||
TrackedObjectType.History => new HistoryState() { InstanceId = key.InstanceId },
|
||||
TrackedObjectType.Instance => new InstanceState() { InstanceId = key.InstanceId },
|
||||
_ => throw new ArgumentException("invalid key", nameof(key)),
|
||||
};
|
||||
|
||||
public static IEnumerable<TrackedObjectKey> GetSingletons()
|
||||
=> Enum.GetValues(typeof(TrackedObjectType)).Cast<TrackedObjectType>().Where(t => IsSingletonType(t)).Select(t => new TrackedObjectKey() { ObjectType = t });
|
||||
|
||||
public override string ToString()
|
||||
=> this.InstanceId == null ? this.ObjectType.ToString() : $"{this.ObjectType}-{this.InstanceId}";
|
||||
|
||||
public void Deserialize(BinaryReader reader)
|
||||
{
|
||||
this.ObjectType = (TrackedObjectType) reader.ReadByte();
|
||||
if (!IsSingletonType(this.ObjectType))
|
||||
{
|
||||
this.InstanceId = reader.ReadString();
|
||||
}
|
||||
}
|
||||
|
||||
public void Serialize(BinaryWriter writer)
|
||||
{
|
||||
writer.Write((byte) this.ObjectType);
|
||||
if (!IsSingletonType(this.ObjectType))
|
||||
{
|
||||
writer.Write(this.InstanceId);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,190 @@
|
|||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
/// <summary>
|
||||
/// Interfaces that separate the transport functionality (which includes both load balancing of partitions
|
||||
/// and transmission of messages) from the host, partition, and client components
|
||||
/// </summary>
|
||||
static class TransportAbstraction
|
||||
{
|
||||
/// <summary>
|
||||
/// The host functionality visible to the transport back-end.
|
||||
/// The transport back-end calls this interface to place clients and partitions on this host.
|
||||
/// </summary>
|
||||
public interface IHost
|
||||
{
|
||||
/// <summary>
|
||||
/// Assigned by the transport backend to inform the host about the number of partitions.
|
||||
/// </summary>
|
||||
uint NumberPartitions { set; }
|
||||
|
||||
/// <summary>
|
||||
/// Returns the storage provider for storing the partition states.
|
||||
/// </summary>
|
||||
IStorageProvider StorageProvider { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Creates a client on this host.
|
||||
/// </summary>
|
||||
/// <param name="clientId">A globally unique identifier for this client</param>
|
||||
/// <param name="taskHubGuid">the unique identifier of the taskhub</param>
|
||||
/// <param name="batchSender">A sender that can be used by the client for sending messages</param>
|
||||
/// <returns>A sender for passing messages to the transport backend</returns>
|
||||
IClient AddClient(Guid clientId, Guid taskHubGuid, ISender batchSender);
|
||||
|
||||
/// <summary>
|
||||
/// Places a partition on this host.
|
||||
/// </summary>
|
||||
/// <param name="partitionId">The partition id.</param>
|
||||
/// <param name="batchSender">A sender for passing messages to the transport backend</param>
|
||||
/// <returns></returns>
|
||||
IPartition AddPartition(uint partitionId, ISender batchSender);
|
||||
|
||||
/// <summary>
|
||||
/// Returns an error handler object for the given partition.
|
||||
/// </summary>
|
||||
/// <param name="partitionId">The partition id.</param>
|
||||
IPartitionErrorHandler CreateErrorHandler(uint partitionId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The partition functionality, as seen by the transport back-end.
|
||||
/// </summary>
|
||||
public interface IPartition
|
||||
{
|
||||
/// <summary>
|
||||
/// The partition id of this partition.
|
||||
/// </summary>
|
||||
uint PartitionId { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Acquire partition ownership, recover partition state from storage, and resume processing.
|
||||
/// </summary>
|
||||
/// <param name="termination">A termination object for initiating and/or detecting termination of the partition.</param>
|
||||
/// <param name="firstInputQueuePosition">For new partitions, the position of the first message to receive.</param>
|
||||
/// <returns>The input queue position of the next message to receive.</returns>
|
||||
/// <remarks>
|
||||
/// The termination token source can be used for immediately terminating the partition.
|
||||
/// Also, it can be used to detect that the partition has terminated for any other reason,
|
||||
/// be it cleanly (after StopAsync) or uncleanly (after losing a lease or hitting a fatal error).
|
||||
/// </remarks>
|
||||
Task<long> CreateOrRestoreAsync(IPartitionErrorHandler termination, long firstInputQueuePosition);
|
||||
|
||||
/// <summary>
|
||||
/// Clean shutdown: stop processing, save partition state to storage, and release ownership.
|
||||
/// </summary>
|
||||
/// <param name="isForced">True if the shutdown should happen as quickly as possible.</param>
|
||||
/// <returns>When all steps have completed and termination is performed.</returns>
|
||||
Task StopAsync(bool isForced);
|
||||
|
||||
/// <summary>
|
||||
/// Queues a single event for processing on this partition.
|
||||
/// </summary>
|
||||
/// <param name="partitionEvent">The event to process.</param>
|
||||
void SubmitInternalEvent(PartitionUpdateEvent partitionEvent);
|
||||
|
||||
/// <summary>
|
||||
/// Queues a batch of incoming external events for processing on this partition.
|
||||
/// </summary>
|
||||
/// <param name="partitionEvents">The events to process.</param>
|
||||
void SubmitExternalEvents(IList<PartitionEvent> partitionEvents);
|
||||
|
||||
/// <summary>
|
||||
/// The error handler for this partition.
|
||||
/// </summary>
|
||||
IPartitionErrorHandler ErrorHandler { get; }
|
||||
|
||||
/// <summary>
|
||||
/// The elapsed time in milliseconds since this partition was constructed. We use this
|
||||
/// mainly for measuring various timings inside a partition.
|
||||
/// </summary>
|
||||
double CurrentTimeMs { get; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The client functionality, as seen by the transport back-end.
|
||||
/// </summary>
|
||||
public interface IClient
|
||||
{
|
||||
/// <summary>
|
||||
/// A unique identifier for this client.
|
||||
/// </summary>
|
||||
Guid ClientId { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Processes a single event on this client.
|
||||
/// </summary>
|
||||
/// <param name="clientEvent">The event to process.</param>
|
||||
void Process(ClientEvent clientEvent);
|
||||
|
||||
/// <summary>
|
||||
/// Stop processing events and shut down.
|
||||
/// </summary>
|
||||
/// <returns>When the client is shut down.</returns>
|
||||
Task StopAsync();
|
||||
|
||||
/// <summary>
|
||||
/// Indicates an observed error for diagnostic purposes.
|
||||
/// </summary>
|
||||
/// <param name="msg">A message describing the circumstances.</param>
|
||||
/// <param name="e">The exception that was observed.</param>
|
||||
void ReportTransportError(string msg, Exception e);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A sender abstraction, passed to clients and partitions, for sending messages via the transport.
|
||||
/// </summary>
|
||||
public interface ISender
|
||||
{
|
||||
/// <summary>
|
||||
/// Send an event. The destination is already determined by the event,
|
||||
/// which contains either a client id or a partition id.
|
||||
/// </summary>
|
||||
/// <param name="element"></param>
|
||||
void Submit(Event element);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A listener abstraction, used by clients and partitions, to receive acks after events have been
|
||||
/// durably processed.
|
||||
/// </summary>
|
||||
public interface IDurabilityListener
|
||||
{
|
||||
/// <summary>
|
||||
/// Indicates that this event has been durably persisted (incoming events) or sent (outgoing events).
|
||||
/// </summary>
|
||||
/// <param name="evt">The event that has been durably processed.</param>
|
||||
void ConfirmDurable(Event evt);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// An <see cref="IDurabilityListener"/> that is also listening for exceptions. Used on the client
|
||||
/// to make transport errors visible to the calling code.
|
||||
/// </summary>
|
||||
public interface IDurabilityOrExceptionListener : IDurabilityListener
|
||||
{
|
||||
/// <summary>
|
||||
/// Indicates that there was an error while trying to send this event.
|
||||
/// </summary>
|
||||
/// <param name="evt"></param>
|
||||
/// <param name="e"></param>
|
||||
void ReportException(Event evt, Exception e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
using System.Runtime.CompilerServices;
|
||||
|
||||
[assembly: InternalsVisibleTo("DurableTask.Netherite.Tests, PublicKey="
|
||||
+ "0024000004800000940000000602000000240000525341310004000001000100e93024fd5f6737"
|
||||
+ "b5f408b39a97e746f673e7c62d690716c6022ee8871b2bc5efcbd8015dd02253302196ded0d9fc"
|
||||
+ "0c6bc0e84740f74ca828fed9aebf15272867019cf991484c06b00c7d7aacb34b40ed0ae633f043"
|
||||
+ "2df41db65caa4b03f9a0e6974c15aeaa7db2acd902914c0628f0b77d9b7839f530ba04584aed92"
|
||||
+ "87159ebd")]
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>netstandard2.1</TargetFramework>
|
||||
<LangVersion>latest</LangVersion>
|
||||
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
|
||||
<GeneratePackageOnBuild>true</GeneratePackageOnBuild>
|
||||
<IncludeSymbols>true</IncludeSymbols>
|
||||
<DebugType>embedded</DebugType>
|
||||
<Description>Netherite durability provider extension for the Durable Task Framework.</Description>
|
||||
<PackageTags>Azure Task Durable Orchestration Workflow Activity Reliable EventHubs</PackageTags>
|
||||
<PackageId>DurableTask.Netherite</PackageId>
|
||||
<PublishRepositoryUrl>true</PublishRepositoryUrl>
|
||||
<EmbedUntrackedSources>true</EmbedUntrackedSources>
|
||||
<SignAssembly>true</SignAssembly>
|
||||
<AssemblyOriginatorKeyFile>..\..\sign.snk</AssemblyOriginatorKeyFile>
|
||||
</PropertyGroup>
|
||||
|
||||
<!-- Version settings: https://andrewlock.net/version-vs-versionsuffix-vs-packageversion-what-do-they-all-mean/ -->
|
||||
<PropertyGroup>
|
||||
<MajorVersion>0</MajorVersion>
|
||||
<VersionPrefix>$(MajorVersion).1.0</VersionPrefix>
|
||||
<VersionSuffix>alpha</VersionSuffix>
|
||||
<AssemblyVersion>$(MajorVersion).0.0.0</AssemblyVersion>
|
||||
<BuildSuffix Condition="'$(GITHUB_RUN_NUMBER)' != ''">.$(GITHUB_RUN_NUMBER)</BuildSuffix>
|
||||
<FileVersion>$(VersionPrefix)$(BuildSuffix)</FileVersion>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">
|
||||
<NoWarn>NU5125;NU5048</NoWarn>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Azure.Cosmos.Table" Version="1.0.8" />
|
||||
<PackageReference Include="Microsoft.Azure.EventHubs.Processor" Version="4.3.0" />
|
||||
<PackageReference Include="Microsoft.Azure.Storage.Blob" Version="11.2.2" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="2.2.0" />
|
||||
<PackageReference Include="Microsoft.FASTER.Core" Version="1.7.2" />
|
||||
<PackageReference Include="Microsoft.Azure.DurableTask.Core" Version="2.4.1" />
|
||||
<PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
|
||||
<PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.0.*" PrivateAssets="All" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using System.Runtime.Serialization;
|
||||
|
||||
[DataContract]
|
||||
abstract class ClientEvent : Event
|
||||
{
|
||||
[DataMember]
|
||||
public Guid ClientId { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public long RequestId { get; set; }
|
||||
|
||||
public override EventId EventId => EventId.MakeClientResponseEventId(this.ClientId, this.RequestId);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System.Runtime.Serialization;
|
||||
using DurableTask.Core;
|
||||
|
||||
[DataContract]
|
||||
class CreationResponseReceived : ClientEvent
|
||||
{
|
||||
[DataMember]
|
||||
public bool Succeeded { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public OrchestrationStatus? ExistingInstanceOrchestrationStatus { get; set; }
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Text;
|
||||
using DurableTask.Core;
|
||||
|
||||
[DataContract]
|
||||
class DeletionResponseReceived : ClientEvent
|
||||
{
|
||||
[DataMember]
|
||||
public int NumberInstancesDeleted { get; set; }
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.Serialization;
|
||||
using DurableTask.Core.History;
|
||||
|
||||
[DataContract]
|
||||
class HistoryResponseReceived : ClientEvent
|
||||
{
|
||||
[DataMember]
|
||||
public string ExecutionId { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public List<HistoryEvent> History { get; set; }
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System.Runtime.Serialization;
|
||||
using System.Text;
|
||||
|
||||
[DataContract]
|
||||
class PurgeResponseReceived : ClientEvent
|
||||
{
|
||||
[DataMember]
|
||||
public int NumberInstancesPurged { get; set; }
|
||||
|
||||
protected override void ExtraTraceInformation(StringBuilder s)
|
||||
{
|
||||
s.Append(" count=");
|
||||
s.Append(this.NumberInstancesPurged);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.Serialization;
|
||||
using DurableTask.Core;
|
||||
|
||||
[DataContract]
|
||||
class QueryResponseReceived : ClientEvent
|
||||
{
|
||||
[DataMember]
|
||||
public IList<OrchestrationState> OrchestrationStates { get; set; }
|
||||
|
||||
public override string ToString() => $"Count: {this.OrchestrationStates.Count}";
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System.Runtime.Serialization;
|
||||
using DurableTask.Core;
|
||||
|
||||
[DataContract]
|
||||
class StateResponseReceived : ClientEvent
|
||||
{
|
||||
[DataMember]
|
||||
public OrchestrationState OrchestrationState { get; set; }
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System.Runtime.Serialization;
|
||||
using System.Text;
|
||||
using DurableTask.Core;
|
||||
|
||||
[DataContract]
|
||||
class WaitResponseReceived : ClientEvent
|
||||
{
|
||||
[DataMember]
|
||||
public OrchestrationState OrchestrationState { get; set; }
|
||||
|
||||
protected override void ExtraTraceInformation(StringBuilder s)
|
||||
{
|
||||
s.Append(' ');
|
||||
s.Append(this.OrchestrationState.OrchestrationStatus.ToString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,106 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Text;
|
||||
|
||||
[DataContract]
|
||||
[KnownTypeAttribute("KnownTypes")]
|
||||
abstract class Event
|
||||
{
|
||||
string eventIdString;
|
||||
|
||||
/// <summary>
|
||||
/// A unique identifier for this event.
|
||||
/// </summary>
|
||||
[IgnoreDataMember]
|
||||
public abstract EventId EventId { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Listeners to be notified when this event is durably persisted or sent.
|
||||
/// </summary>
|
||||
[IgnoreDataMember]
|
||||
public DurabilityListeners DurabilityListeners;
|
||||
|
||||
/// <summary>
|
||||
/// A string identifiying this event, suitable for tracing (cached to avoid excessive formatting)
|
||||
/// </summary>
|
||||
[IgnoreDataMember]
|
||||
public string EventIdString => this.eventIdString ?? (this.eventIdString = this.EventId.ToString());
|
||||
|
||||
public override string ToString()
|
||||
{
|
||||
var s = new StringBuilder();
|
||||
s.Append(this.GetType().Name);
|
||||
this.ExtraTraceInformation(s);
|
||||
return s.ToString();
|
||||
}
|
||||
|
||||
protected virtual void ExtraTraceInformation(StringBuilder s)
|
||||
{
|
||||
// subclasses can override this to add extra information to the trace
|
||||
}
|
||||
|
||||
public static IEnumerable<Type> KnownTypes()
|
||||
{
|
||||
yield return typeof(ClientEventFragment);
|
||||
yield return typeof(CreationResponseReceived);
|
||||
yield return typeof(DeletionResponseReceived);
|
||||
yield return typeof(HistoryResponseReceived);
|
||||
yield return typeof(PurgeResponseReceived);
|
||||
yield return typeof(QueryResponseReceived);
|
||||
yield return typeof(StateResponseReceived);
|
||||
yield return typeof(WaitResponseReceived);
|
||||
yield return typeof(ClientTaskMessagesReceived);
|
||||
yield return typeof(CreationRequestReceived);
|
||||
yield return typeof(DeletionRequestReceived);
|
||||
yield return typeof(HistoryRequestReceived);
|
||||
yield return typeof(InstanceQueryReceived);
|
||||
yield return typeof(PurgeRequestReceived);
|
||||
yield return typeof(StateRequestReceived);
|
||||
yield return typeof(WaitRequestReceived);
|
||||
yield return typeof(ActivityCompleted);
|
||||
yield return typeof(BatchProcessed);
|
||||
yield return typeof(SendConfirmed);
|
||||
yield return typeof(TimerFired);
|
||||
yield return typeof(ActivityOffloadReceived);
|
||||
yield return typeof(RemoteActivityResultReceived);
|
||||
yield return typeof(TaskMessagesReceived);
|
||||
yield return typeof(OffloadDecision);
|
||||
yield return typeof(PurgeBatchIssued);
|
||||
yield return typeof(PartitionEventFragment);
|
||||
}
|
||||
|
||||
public bool SafeToRetryFailedSend()
|
||||
{
|
||||
if (this is ClientRequestEvent)
|
||||
{
|
||||
// these are not safe to duplicate as they could restart an orchestration or deliver a message twice
|
||||
return false;
|
||||
}
|
||||
else
|
||||
{
|
||||
// all others are safe to duplicate:
|
||||
// - duplicate responses sent to clients are simply ignored
|
||||
// - duplicate read requests cause no harm (other than redundant work)
|
||||
// - duplicate partition events are deduplicated by Dedup
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,146 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Text;
|
||||
using DurableTask.Core;
|
||||
using Dynamitey;
|
||||
|
||||
/// <summary>
|
||||
/// A unique identifier for an event.
|
||||
/// </summary>
|
||||
public struct EventId
|
||||
{
|
||||
/// <summary>
|
||||
/// The category of an event.
|
||||
/// </summary>
|
||||
public enum EventCategory
|
||||
{
|
||||
/// <summary>
|
||||
/// An event that is sent from a client to a partition.
|
||||
/// </summary>
|
||||
ClientRequest,
|
||||
|
||||
/// <summary>
|
||||
/// An event that is sent from a partition back to a client, as a response.
|
||||
/// </summary>
|
||||
ClientResponse,
|
||||
|
||||
/// <summary>
|
||||
/// An event that is sent by a partition to itself.
|
||||
/// </summary>
|
||||
PartitionInternal,
|
||||
|
||||
/// <summary>
|
||||
/// An event that is sent from a partition to another partition.
|
||||
/// </summary>
|
||||
PartitionToPartition,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The category of this event
|
||||
/// </summary>
|
||||
public EventCategory Category { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// For events originating on a client, the client id.
|
||||
/// </summary>
|
||||
public Guid ClientId { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// For events originating on a partition, the partition id.
|
||||
/// </summary>
|
||||
public uint PartitionId { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// For events originating on a client, a sequence number
|
||||
/// </summary>
|
||||
public long Number { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// For sub-events, the index
|
||||
/// </summary>
|
||||
public int SubIndex { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// For events originating on a partition, a string for correlating this event
|
||||
/// </summary>
|
||||
public string WorkItemId { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// For fragmented events, or internal dependent reads, the fragment number or subindex.
|
||||
/// </summary>
|
||||
public int? Index { get; set; }
|
||||
|
||||
internal static EventId MakeClientRequestEventId(Guid ClientId, long RequestId) => new EventId()
|
||||
{
|
||||
ClientId = ClientId,
|
||||
Number = RequestId,
|
||||
Category = EventCategory.ClientRequest
|
||||
};
|
||||
|
||||
internal static EventId MakeClientResponseEventId(Guid ClientId, long RequestId) => new EventId()
|
||||
{
|
||||
ClientId = ClientId,
|
||||
Number = RequestId,
|
||||
Category = EventCategory.ClientResponse
|
||||
};
|
||||
|
||||
internal static EventId MakePartitionInternalEventId(string workItemId) => new EventId()
|
||||
{
|
||||
WorkItemId = workItemId,
|
||||
Category = EventCategory.PartitionInternal
|
||||
};
|
||||
|
||||
internal static EventId MakePartitionToPartitionEventId(string workItemId, uint destinationPartition) => new EventId()
|
||||
{
|
||||
WorkItemId = workItemId,
|
||||
PartitionId = destinationPartition,
|
||||
Category = EventCategory.PartitionToPartition
|
||||
};
|
||||
|
||||
internal static EventId MakeSubEventId(EventId id, int fragment)
|
||||
{
|
||||
id.Index = fragment;
|
||||
return id;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public override string ToString()
|
||||
{
|
||||
switch (this.Category)
|
||||
{
|
||||
case EventCategory.ClientRequest:
|
||||
return $"{Client.GetShortId(this.ClientId)}-{this.Number}{this.IndexSuffix}";
|
||||
|
||||
case EventCategory.ClientResponse:
|
||||
return $"{Client.GetShortId(this.ClientId)}-{this.Number}R{this.IndexSuffix}";
|
||||
|
||||
case EventCategory.PartitionInternal:
|
||||
return $"{this.WorkItemId}{this.IndexSuffix}";
|
||||
|
||||
case EventCategory.PartitionToPartition:
|
||||
return $"{this.WorkItemId}-{this.PartitionId:D2}{this.IndexSuffix}";
|
||||
|
||||
default:
|
||||
throw new InvalidOperationException();
|
||||
}
|
||||
}
|
||||
|
||||
string IndexSuffix => this.Index.HasValue ? $"-{this.Index.Value}" : string.Empty;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System.Runtime.Serialization;
|
||||
|
||||
[DataContract]
|
||||
class ClientEventFragment : ClientEvent, FragmentationAndReassembly.IEventFragment
|
||||
{
|
||||
[DataMember]
|
||||
public EventId OriginalEventId { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public byte[] Bytes { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public int Fragment { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public bool IsLast { get; set; }
|
||||
|
||||
public override EventId EventId => EventId.MakeSubEventId(this.OriginalEventId, this.Fragment);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System.Runtime.Serialization;
|
||||
using System.Text;
|
||||
|
||||
[DataContract]
|
||||
class PartitionEventFragment :
|
||||
PartitionUpdateEvent,
|
||||
FragmentationAndReassembly.IEventFragment
|
||||
{
|
||||
[DataMember]
|
||||
public EventId OriginalEventId { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public byte[] Bytes { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public int Fragment { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public bool IsLast { get; set; }
|
||||
|
||||
[IgnoreDataMember]
|
||||
public PartitionEvent ReassembledEvent;
|
||||
|
||||
public override EventId EventId => EventId.MakeSubEventId(this.OriginalEventId, this.Fragment);
|
||||
|
||||
protected override void ExtraTraceInformation(StringBuilder s)
|
||||
{
|
||||
s.Append(' ');
|
||||
s.Append(this.Bytes.Length);
|
||||
if (this.IsLast)
|
||||
{
|
||||
s.Append(" last");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public override void DetermineEffects(EffectTracker effects)
|
||||
{
|
||||
effects.Add(TrackedObjectKey.Reassembly);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,104 @@
|
|||
namespace DurableTask.Netherite
|
||||
{
|
||||
using DurableTask.Core.Common;
|
||||
using DurableTask.Core.Exceptions;
|
||||
using Dynamitey;
|
||||
using Newtonsoft.Json;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Data;
|
||||
using System.IO;
|
||||
using System.Reflection.Emit;
|
||||
using System.Text;
|
||||
|
||||
/// <summary>
|
||||
/// Packets are the unit of transmission for external events, among clients and partitions.
|
||||
/// </summary>
|
||||
static class Packet
|
||||
{
|
||||
// we prefix packets with a byte indicating the format
|
||||
// we use a flag (for json) or a version (for binary) to facilitate changing formats in the future
|
||||
static readonly byte jsonVersion = 0;
|
||||
static readonly byte binaryVersion = 1;
|
||||
|
||||
static readonly JsonSerializerSettings serializerSettings
|
||||
= new JsonSerializerSettings() { TypeNameHandling = TypeNameHandling.Auto };
|
||||
|
||||
public static void Serialize(Event evt, Stream stream, bool useJson, byte[] taskHubGuid)
|
||||
{
|
||||
var writer = new BinaryWriter(stream, Encoding.UTF8);
|
||||
|
||||
if (useJson)
|
||||
{
|
||||
// serialize the json
|
||||
string jsonContent = JsonConvert.SerializeObject(evt, typeof(Event), Packet.serializerSettings);
|
||||
|
||||
// first entry is the version and the taskhub
|
||||
writer.Write(Packet.jsonVersion);
|
||||
writer.Write(taskHubGuid);
|
||||
|
||||
// then we write the json string
|
||||
writer.Write(jsonContent);
|
||||
}
|
||||
else
|
||||
{
|
||||
// first entry is the version and the taskhub
|
||||
writer.Write(Packet.binaryVersion);
|
||||
writer.Write(taskHubGuid);
|
||||
|
||||
writer.Flush();
|
||||
|
||||
// then we write the binary serialization to the stream
|
||||
Serializer.SerializeEvent(evt, stream);
|
||||
}
|
||||
}
|
||||
|
||||
public static void Deserialize<TEvent>(Stream stream, out TEvent evt, byte[] taskHubGuid) where TEvent : Event
|
||||
{
|
||||
var reader = new BinaryReader(stream);
|
||||
var format = reader.ReadByte();
|
||||
var destinationTaskHubGuid = reader.ReadBytes(16);
|
||||
|
||||
if (taskHubGuid != null && !GuidMatches(taskHubGuid, destinationTaskHubGuid))
|
||||
{
|
||||
evt = null;
|
||||
return;
|
||||
}
|
||||
|
||||
if (format == Packet.jsonVersion)
|
||||
{
|
||||
string jsonContent = reader.ReadString();
|
||||
evt = (TEvent)JsonConvert.DeserializeObject(jsonContent, Packet.serializerSettings);
|
||||
}
|
||||
else if (format == Packet.binaryVersion)
|
||||
{
|
||||
evt = (TEvent)Serializer.DeserializeEvent(stream);
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new VersionNotFoundException($"Received packet with unhandled format indicator {format} - likely a versioning issue");
|
||||
}
|
||||
}
|
||||
|
||||
public static void Deserialize<TEvent>(ArraySegment<byte> arraySegment, out TEvent evt, byte[] taskHubGuid) where TEvent : Event
|
||||
{
|
||||
using (var stream = new MemoryStream(arraySegment.Array, arraySegment.Offset, arraySegment.Count, false))
|
||||
{
|
||||
Packet.Deserialize(stream, out evt, taskHubGuid);
|
||||
}
|
||||
}
|
||||
|
||||
public static bool GuidMatches(byte[] expected, byte[] actual)
|
||||
{
|
||||
for (int i = 0; i < 16; i++)
|
||||
{
|
||||
if (expected[i] != actual[i])
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
32
src/DurableTask.Netherite/Events/PartitionEvents/External/FromClients/ClientReadonlyRequestEvent.cs
поставляемый
Normal file
32
src/DurableTask.Netherite/Events/PartitionEvents/External/FromClients/ClientReadonlyRequestEvent.cs
поставляемый
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Text;
|
||||
|
||||
[DataContract]
|
||||
abstract class ClientReadonlyRequestEvent : PartitionReadEvent, IClientRequestEvent
|
||||
{
|
||||
[DataMember]
|
||||
public Guid ClientId { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public long RequestId { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public DateTime TimeoutUtc { get; set; }
|
||||
|
||||
[IgnoreDataMember]
|
||||
public override EventId EventId => EventId.MakeClientRequestEventId(this.ClientId, this.RequestId);
|
||||
}
|
||||
}
|
||||
32
src/DurableTask.Netherite/Events/PartitionEvents/External/FromClients/ClientRequestEvent.cs
поставляемый
Normal file
32
src/DurableTask.Netherite/Events/PartitionEvents/External/FromClients/ClientRequestEvent.cs
поставляемый
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Text;
|
||||
|
||||
[DataContract]
|
||||
abstract class ClientRequestEvent : PartitionUpdateEvent, IClientRequestEvent
|
||||
{
|
||||
[DataMember]
|
||||
public Guid ClientId { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public long RequestId { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public DateTime TimeoutUtc { get; set; }
|
||||
|
||||
[IgnoreDataMember]
|
||||
public override EventId EventId => EventId.MakeClientRequestEventId(this.ClientId, this.RequestId);
|
||||
}
|
||||
}
|
||||
46
src/DurableTask.Netherite/Events/PartitionEvents/External/FromClients/ClientRequestEventWithPrefetch.cs
поставляемый
Normal file
46
src/DurableTask.Netherite/Events/PartitionEvents/External/FromClients/ClientRequestEventWithPrefetch.cs
поставляемый
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Text;
|
||||
|
||||
[DataContract]
|
||||
abstract class ClientRequestEventWithPrefetch : ClientRequestEvent, IClientRequestEvent
|
||||
{
|
||||
[DataMember]
|
||||
public ProcessingPhase Phase { get; set; }
|
||||
|
||||
[IgnoreDataMember]
|
||||
public abstract TrackedObjectKey Target { get; }
|
||||
|
||||
public virtual bool OnReadComplete(TrackedObject target, Partition partition)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
[IgnoreDataMember]
|
||||
public virtual TrackedObjectKey? Prefetch => null;
|
||||
|
||||
public sealed override void DetermineEffects(EffectTracker effects)
|
||||
{
|
||||
effects.Add(TrackedObjectKey.Prefetch);
|
||||
}
|
||||
|
||||
public enum ProcessingPhase
|
||||
{
|
||||
Read,
|
||||
Confirm,
|
||||
ConfirmAndProcess,
|
||||
}
|
||||
}
|
||||
}
|
||||
45
src/DurableTask.Netherite/Events/PartitionEvents/External/FromClients/ClientRequestEventWithQuery.cs
поставляемый
Normal file
45
src/DurableTask.Netherite/Events/PartitionEvents/External/FromClients/ClientRequestEventWithQuery.cs
поставляемый
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using DurableTask.Core;
|
||||
|
||||
[DataContract]
|
||||
abstract class ClientRequestEventWithQuery : ClientRequestEvent, IClientRequestEvent
|
||||
{
|
||||
[DataMember]
|
||||
public ProcessingPhase Phase { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public InstanceQuery InstanceQuery { get; set; }
|
||||
|
||||
[IgnoreDataMember]
|
||||
public override EventId EventId => EventId.MakeClientRequestEventId(this.ClientId, this.RequestId);
|
||||
|
||||
public abstract Task OnQueryCompleteAsync(IAsyncEnumerable<OrchestrationState> result, Partition partition);
|
||||
|
||||
public sealed override void DetermineEffects(EffectTracker effects)
|
||||
{
|
||||
effects.Add(TrackedObjectKey.Queries);
|
||||
}
|
||||
|
||||
public enum ProcessingPhase
|
||||
{
|
||||
Query,
|
||||
Confirm,
|
||||
ConfirmAndProcess,
|
||||
}
|
||||
}
|
||||
}
|
||||
34
src/DurableTask.Netherite/Events/PartitionEvents/External/FromClients/ClientTaskMessagesReceived.cs
поставляемый
Normal file
34
src/DurableTask.Netherite/Events/PartitionEvents/External/FromClients/ClientTaskMessagesReceived.cs
поставляемый
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.Serialization;
|
||||
using DurableTask.Core;
|
||||
|
||||
[DataContract]
|
||||
class ClientTaskMessagesReceived : ClientRequestEvent
|
||||
{
|
||||
[DataMember]
|
||||
public TaskMessage[] TaskMessages { get; set; }
|
||||
|
||||
[IgnoreDataMember]
|
||||
public override IEnumerable<TaskMessage> TracedTaskMessages => this.TaskMessages;
|
||||
|
||||
public override void DetermineEffects(EffectTracker effects)
|
||||
{
|
||||
effects.Add(TrackedObjectKey.Sessions);
|
||||
}
|
||||
}
|
||||
}
|
||||
65
src/DurableTask.Netherite/Events/PartitionEvents/External/FromClients/CreationRequestReceived.cs
поставляемый
Normal file
65
src/DurableTask.Netherite/Events/PartitionEvents/External/FromClients/CreationRequestReceived.cs
поставляемый
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.Serialization;
|
||||
using DurableTask.Core;
|
||||
using DurableTask.Core.History;
|
||||
|
||||
[DataContract]
|
||||
class CreationRequestReceived : ClientRequestEventWithPrefetch
|
||||
{
|
||||
[DataMember]
|
||||
public OrchestrationStatus[] DedupeStatuses { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public DateTime Timestamp { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public TaskMessage TaskMessage { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public bool FilteredDuplicate { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public OrchestrationStatus? ExistingInstanceOrchestrationStatus { get; set; }
|
||||
|
||||
[IgnoreDataMember]
|
||||
public ExecutionStartedEvent ExecutionStartedEvent => this.TaskMessage.Event as ExecutionStartedEvent;
|
||||
|
||||
[IgnoreDataMember]
|
||||
public string InstanceId => this.ExecutionStartedEvent.OrchestrationInstance.InstanceId;
|
||||
|
||||
[IgnoreDataMember]
|
||||
public override IEnumerable<TaskMessage> TracedTaskMessages { get { yield return this.TaskMessage; } }
|
||||
|
||||
[IgnoreDataMember]
|
||||
public override TrackedObjectKey Target => TrackedObjectKey.Instance(this.InstanceId);
|
||||
|
||||
|
||||
public override bool OnReadComplete(TrackedObject target, Partition partition)
|
||||
{
|
||||
// Use this moment of time as the creation timestamp, replacing the original timestamp taken on the client.
|
||||
// This is preferrable because it avoids clock synchronization issues (which can result in negative orchestration durations)
|
||||
// and means the timestamp is consistently ordered with respect to timestamps of other events on this partition.
|
||||
DateTime creationTimestamp = DateTime.UtcNow;
|
||||
|
||||
this.ExecutionStartedEvent.Timestamp = creationTimestamp;
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
41
src/DurableTask.Netherite/Events/PartitionEvents/External/FromClients/DeletionRequestReceived.cs
поставляемый
Normal file
41
src/DurableTask.Netherite/Events/PartitionEvents/External/FromClients/DeletionRequestReceived.cs
поставляемый
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Text;
|
||||
|
||||
[DataContract]
|
||||
class DeletionRequestReceived : ClientRequestEventWithPrefetch
|
||||
{
|
||||
[DataMember]
|
||||
public string InstanceId { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public DateTime? CreatedTime { get; set; } // works like an e-tag, if specified
|
||||
|
||||
[IgnoreDataMember]
|
||||
public override TrackedObjectKey Target => TrackedObjectKey.Instance(this.InstanceId);
|
||||
|
||||
[IgnoreDataMember]
|
||||
public override TrackedObjectKey? Prefetch => TrackedObjectKey.History(this.InstanceId);
|
||||
|
||||
protected override void ExtraTraceInformation(StringBuilder s)
|
||||
{
|
||||
s.Append(' ');
|
||||
s.Append(this.InstanceId);
|
||||
}
|
||||
}
|
||||
}
|
||||
50
src/DurableTask.Netherite/Events/PartitionEvents/External/FromClients/HistoryRequestReceived.cs
поставляемый
Normal file
50
src/DurableTask.Netherite/Events/PartitionEvents/External/FromClients/HistoryRequestReceived.cs
поставляемый
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System.Linq;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Text;
|
||||
|
||||
[DataContract]
|
||||
class HistoryRequestReceived : ClientReadonlyRequestEvent
|
||||
{
|
||||
[DataMember]
|
||||
public string InstanceId { get; set; }
|
||||
|
||||
[IgnoreDataMember]
|
||||
public override TrackedObjectKey ReadTarget => TrackedObjectKey.History(this.InstanceId);
|
||||
|
||||
protected override void ExtraTraceInformation(StringBuilder s)
|
||||
{
|
||||
s.Append(' ');
|
||||
s.Append(this.InstanceId);
|
||||
}
|
||||
|
||||
public override void OnReadComplete(TrackedObject target, Partition partition)
|
||||
{
|
||||
var historyState = (HistoryState)target;
|
||||
|
||||
var response = new HistoryResponseReceived()
|
||||
{
|
||||
ClientId = this.ClientId,
|
||||
RequestId = this.RequestId,
|
||||
ExecutionId = historyState?.ExecutionId,
|
||||
History = historyState?.History?.ToList(),
|
||||
};
|
||||
|
||||
partition.Send(response);
|
||||
}
|
||||
}
|
||||
}
|
||||
24
src/DurableTask.Netherite/Events/PartitionEvents/External/FromClients/IClientRequestEvent.cs
поставляемый
Normal file
24
src/DurableTask.Netherite/Events/PartitionEvents/External/FromClients/IClientRequestEvent.cs
поставляемый
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
|
||||
interface IClientRequestEvent
|
||||
{
|
||||
Guid ClientId { get; set; }
|
||||
|
||||
long RequestId { get; set; }
|
||||
|
||||
DateTime TimeoutUtc { get; set; }
|
||||
|
||||
EventId EventId { get; }
|
||||
}
|
||||
}
|
||||
41
src/DurableTask.Netherite/Events/PartitionEvents/External/FromClients/InstanceQueryReceived.cs
поставляемый
Normal file
41
src/DurableTask.Netherite/Events/PartitionEvents/External/FromClients/InstanceQueryReceived.cs
поставляемый
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Threading.Tasks;
|
||||
using DurableTask.Core;
|
||||
|
||||
[DataContract]
|
||||
class InstanceQueryReceived : ClientRequestEventWithQuery
|
||||
{
|
||||
public async override Task OnQueryCompleteAsync(IAsyncEnumerable<OrchestrationState> instances, Partition partition)
|
||||
{
|
||||
var response = new QueryResponseReceived
|
||||
{
|
||||
ClientId = this.ClientId,
|
||||
RequestId = this.RequestId,
|
||||
OrchestrationStates = new List<OrchestrationState>()
|
||||
};
|
||||
|
||||
await foreach (var orchestrationState in instances)
|
||||
{
|
||||
response.OrchestrationStates.Add(orchestrationState);
|
||||
}
|
||||
|
||||
partition.Send(response);
|
||||
}
|
||||
}
|
||||
}
|
||||
75
src/DurableTask.Netherite/Events/PartitionEvents/External/FromClients/PurgeRequestReceived.cs
поставляемый
Normal file
75
src/DurableTask.Netherite/Events/PartitionEvents/External/FromClients/PurgeRequestReceived.cs
поставляемый
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Threading.Tasks;
|
||||
using DurableTask.Core;
|
||||
|
||||
[DataContract]
|
||||
class PurgeRequestReceived : ClientRequestEventWithQuery
|
||||
{
|
||||
// tracks total number of purges that happened
|
||||
// it is stored in QueriesState so it correctly counts all successful purges
|
||||
// even if a query is restarted before completing
|
||||
[DataMember]
|
||||
public int NumberInstancesPurged { get; set; } = 0;
|
||||
|
||||
const int MaxBatchSize = 1000;
|
||||
|
||||
public async override Task OnQueryCompleteAsync(IAsyncEnumerable<OrchestrationState> instances, Partition partition)
|
||||
{
|
||||
int batchCount = 0;
|
||||
|
||||
PurgeBatchIssued makeBatchObject()
|
||||
=> new PurgeBatchIssued()
|
||||
{
|
||||
PartitionId = partition.PartitionId,
|
||||
QueryEventId = this.EventIdString,
|
||||
BatchNumber = batchCount++,
|
||||
InstanceIds = new List<string>(),
|
||||
WhenProcessed = new TaskCompletionSource<object>(),
|
||||
InstanceQuery = this.InstanceQuery,
|
||||
};
|
||||
|
||||
PurgeBatchIssued batch = makeBatchObject();
|
||||
|
||||
await foreach (var orchestrationState in instances)
|
||||
{
|
||||
batch.InstanceIds.Add(orchestrationState.OrchestrationInstance.InstanceId);
|
||||
|
||||
if (batch.InstanceIds.Count == MaxBatchSize)
|
||||
{
|
||||
partition.SubmitInternalEvent(batch);
|
||||
await batch.WhenProcessed.Task;
|
||||
makeBatchObject();
|
||||
}
|
||||
}
|
||||
|
||||
if (batch.InstanceIds.Count > 0)
|
||||
{
|
||||
partition.SubmitInternalEvent(batch);
|
||||
await batch.WhenProcessed.Task;
|
||||
}
|
||||
|
||||
partition.Send(new PurgeResponseReceived()
|
||||
{
|
||||
ClientId = this.ClientId,
|
||||
RequestId = this.RequestId,
|
||||
NumberInstancesPurged = this.NumberInstancesPurged,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
55
src/DurableTask.Netherite/Events/PartitionEvents/External/FromClients/StateRequestReceived.cs
поставляемый
Normal file
55
src/DurableTask.Netherite/Events/PartitionEvents/External/FromClients/StateRequestReceived.cs
поставляемый
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System.Runtime.Serialization;
|
||||
using System.Text;
|
||||
|
||||
[DataContract]
|
||||
class StateRequestReceived : ClientReadonlyRequestEvent
|
||||
{
|
||||
[DataMember]
|
||||
public string InstanceId { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public bool IncludeInput { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public bool IncludeOutput { get; set; }
|
||||
|
||||
[IgnoreDataMember]
|
||||
public override TrackedObjectKey ReadTarget => TrackedObjectKey.Instance(this.InstanceId);
|
||||
|
||||
protected override void ExtraTraceInformation(StringBuilder s)
|
||||
{
|
||||
s.Append(' ');
|
||||
s.Append(this.InstanceId);
|
||||
}
|
||||
|
||||
public override void OnReadComplete(TrackedObject target, Partition partition)
|
||||
{
|
||||
var orchestrationState = ((InstanceState)target)?.OrchestrationState;
|
||||
var editedState = orchestrationState?.ClearFieldsImmutably(this.IncludeInput, this.IncludeOutput);
|
||||
|
||||
var response = new StateResponseReceived()
|
||||
{
|
||||
ClientId = this.ClientId,
|
||||
RequestId = this.RequestId,
|
||||
OrchestrationState = editedState,
|
||||
};
|
||||
|
||||
partition.Send(response);
|
||||
}
|
||||
}
|
||||
}
|
||||
47
src/DurableTask.Netherite/Events/PartitionEvents/External/FromClients/WaitRequestReceived.cs
поставляемый
Normal file
47
src/DurableTask.Netherite/Events/PartitionEvents/External/FromClients/WaitRequestReceived.cs
поставляемый
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using DurableTask.Core;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Text;
|
||||
|
||||
[DataContract]
|
||||
class WaitRequestReceived : ClientRequestEventWithPrefetch
|
||||
{
|
||||
[DataMember]
|
||||
public string InstanceId { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public string ExecutionId { get; set; }
|
||||
|
||||
public override TrackedObjectKey Target => TrackedObjectKey.Instance(this.InstanceId);
|
||||
|
||||
protected override void ExtraTraceInformation(StringBuilder s)
|
||||
{
|
||||
s.Append(' ');
|
||||
s.Append(this.InstanceId);
|
||||
}
|
||||
|
||||
public static bool SatisfiesWaitCondition(OrchestrationState value)
|
||||
=> (value != null &&
|
||||
value.OrchestrationStatus != OrchestrationStatus.Running &&
|
||||
value.OrchestrationStatus != OrchestrationStatus.Pending &&
|
||||
value.OrchestrationStatus != OrchestrationStatus.ContinuedAsNew);
|
||||
|
||||
public WaitResponseReceived CreateResponse(OrchestrationState value)
|
||||
=> new WaitResponseReceived()
|
||||
{
|
||||
ClientId = this.ClientId,
|
||||
RequestId = this.RequestId,
|
||||
OrchestrationState = value
|
||||
};
|
||||
}
|
||||
}
|
||||
36
src/DurableTask.Netherite/Events/PartitionEvents/External/FromPartitions/ActivityOffloadReceived.cs
поставляемый
Normal file
36
src/DurableTask.Netherite/Events/PartitionEvents/External/FromPartitions/ActivityOffloadReceived.cs
поставляемый
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.Serialization;
|
||||
using DurableTask.Core;
|
||||
|
||||
[DataContract]
|
||||
class ActivityOffloadReceived : PartitionMessageEvent
|
||||
{
|
||||
[DataMember]
|
||||
public List<TaskMessage> OffloadedActivities { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public DateTime Timestamp { get; set; }
|
||||
|
||||
[IgnoreDataMember]
|
||||
public override EventId EventId => EventId.MakePartitionToPartitionEventId(OffloadDecision.GetWorkItemId(this.OriginPartition, this.Timestamp), this.PartitionId);
|
||||
|
||||
[IgnoreDataMember]
|
||||
public override IEnumerable<TaskMessage> TracedTaskMessages => this.OffloadedActivities;
|
||||
}
|
||||
}
|
||||
32
src/DurableTask.Netherite/Events/PartitionEvents/External/FromPartitions/PartitionMessageEvent.cs
поставляемый
Normal file
32
src/DurableTask.Netherite/Events/PartitionEvents/External/FromPartitions/PartitionMessageEvent.cs
поставляемый
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System.Runtime.Serialization;
|
||||
|
||||
[DataContract]
|
||||
abstract class PartitionMessageEvent : PartitionUpdateEvent
|
||||
{
|
||||
[DataMember]
|
||||
public uint OriginPartition { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public long OriginPosition { get; set; }
|
||||
|
||||
public override void DetermineEffects(EffectTracker effects)
|
||||
{
|
||||
effects.Add(TrackedObjectKey.Dedup);
|
||||
}
|
||||
}
|
||||
}
|
||||
45
src/DurableTask.Netherite/Events/PartitionEvents/External/FromPartitions/RemoteActivityResultReceived.cs
поставляемый
Normal file
45
src/DurableTask.Netherite/Events/PartitionEvents/External/FromPartitions/RemoteActivityResultReceived.cs
поставляемый
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Text;
|
||||
using DurableTask.Core;
|
||||
using DurableTask.Core.Exceptions;
|
||||
using DurableTask.Core.History;
|
||||
|
||||
[DataContract]
|
||||
class RemoteActivityResultReceived : PartitionMessageEvent
|
||||
{
|
||||
[DataMember]
|
||||
public TaskMessage Result { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public long ActivityId { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public int ActivitiesQueueSize { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public DateTime Timestamp { get; set; }
|
||||
|
||||
[IgnoreDataMember]
|
||||
public override EventId EventId => EventId.MakePartitionToPartitionEventId(ActivitiesState.GetWorkItemId(this.OriginPartition, this.ActivityId), this.PartitionId);
|
||||
|
||||
[IgnoreDataMember]
|
||||
public override IEnumerable<TaskMessage> TracedTaskMessages { get { yield return this.Result; } }
|
||||
}
|
||||
}
|
||||
58
src/DurableTask.Netherite/Events/PartitionEvents/External/FromPartitions/TaskMessagesReceived.cs
поставляемый
Normal file
58
src/DurableTask.Netherite/Events/PartitionEvents/External/FromPartitions/TaskMessagesReceived.cs
поставляемый
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Text;
|
||||
using DurableTask.Core;
|
||||
|
||||
[DataContract]
|
||||
class TaskMessagesReceived : PartitionMessageEvent
|
||||
{
|
||||
[DataMember]
|
||||
public List<TaskMessage> TaskMessages { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public List<TaskMessage> DelayedTaskMessages { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public string WorkItemId { get; set; }
|
||||
|
||||
[IgnoreDataMember]
|
||||
public override EventId EventId => EventId.MakePartitionToPartitionEventId(this.WorkItemId, this.PartitionId);
|
||||
|
||||
protected override void ExtraTraceInformation(StringBuilder s)
|
||||
{
|
||||
var tCount = this.TaskMessages?.Count ?? 0;
|
||||
var dCount = this.DelayedTaskMessages?.Count ?? 0;
|
||||
|
||||
s.Append(' ');
|
||||
if (tCount == 1)
|
||||
{
|
||||
s.Append(this.TaskMessages[0].Event.EventType);
|
||||
}
|
||||
else if (dCount == 1)
|
||||
{
|
||||
s.Append(this.DelayedTaskMessages[0].Event.EventType);
|
||||
}
|
||||
else
|
||||
{
|
||||
s.Append('[');
|
||||
s.Append(tCount + dCount);
|
||||
s.Append(']');
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.Serialization;
|
||||
using DurableTask.Core;
|
||||
|
||||
[DataContract]
|
||||
class ActivityCompleted : PartitionUpdateEvent
|
||||
{
|
||||
[DataMember]
|
||||
public long ActivityId { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public TaskMessage Response { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public DateTime Timestamp { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public uint OriginPartitionId { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public int ReportedLoad { get; set; }
|
||||
|
||||
[IgnoreDataMember]
|
||||
public override EventId EventId => EventId.MakePartitionInternalEventId(ActivitiesState.GetWorkItemId(this.PartitionId, this.ActivityId));
|
||||
|
||||
[IgnoreDataMember]
|
||||
public override IEnumerable<TaskMessage> TracedTaskMessages { get { yield return this.Response; } }
|
||||
|
||||
public override void DetermineEffects(EffectTracker effects)
|
||||
{
|
||||
effects.Add(TrackedObjectKey.Activities);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,109 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Text;
|
||||
using DurableTask.Core;
|
||||
using DurableTask.Core.History;
|
||||
|
||||
[DataContract]
|
||||
class BatchProcessed : PartitionUpdateEvent
|
||||
{
|
||||
[DataMember]
|
||||
public long SessionId { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public string InstanceId { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public long BatchStartPosition { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public int BatchLength { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public List<HistoryEvent> NewEvents { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public OrchestrationState State { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public List<TaskMessage> ActivityMessages { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public List<TaskMessage> LocalMessages { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public List<TaskMessage> RemoteMessages { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public List<TaskMessage> TimerMessages { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public DateTime Timestamp { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public bool IsPersisted { get; set; }
|
||||
|
||||
[IgnoreDataMember]
|
||||
public OrchestrationWorkItem WorkItemForReuse { get; set; }
|
||||
|
||||
[IgnoreDataMember]
|
||||
public string WorkItemId => SessionsState.GetWorkItemId(this.PartitionId, this.SessionId, this.BatchStartPosition);
|
||||
|
||||
[IgnoreDataMember]
|
||||
public override EventId EventId => EventId.MakePartitionInternalEventId(this.IsPersisted ? this.WorkItemId + "P" : this.WorkItemId);
|
||||
|
||||
[IgnoreDataMember]
|
||||
public override IEnumerable<TaskMessage> TracedTaskMessages
|
||||
{
|
||||
get
|
||||
{
|
||||
if (this.ActivityMessages != null)
|
||||
foreach (var a in this.ActivityMessages)
|
||||
yield return a;
|
||||
if (this.LocalMessages != null)
|
||||
foreach (var l in this.LocalMessages)
|
||||
yield return l;
|
||||
if (this.RemoteMessages != null)
|
||||
foreach (var r in this.RemoteMessages)
|
||||
yield return r;
|
||||
// we are not including the timer messages because they are considered "sent" at the time the timer fires, not when it is scheduled
|
||||
}
|
||||
}
|
||||
|
||||
public override void DetermineEffects(EffectTracker effects)
|
||||
{
|
||||
// start on the sessions object; further effects are determined from there
|
||||
effects.Add(TrackedObjectKey.Sessions);
|
||||
}
|
||||
|
||||
protected override void ExtraTraceInformation(StringBuilder s)
|
||||
{
|
||||
base.ExtraTraceInformation(s);
|
||||
|
||||
if (this.State != null)
|
||||
{
|
||||
s.Append(' ');
|
||||
s.Append(this.State.OrchestrationStatus);
|
||||
}
|
||||
|
||||
s.Append(' ');
|
||||
s.Append(this.InstanceId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
abstract class InternalReadEvent : PartitionReadEvent
|
||||
{
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.Serialization;
|
||||
using DurableTask.Core;
|
||||
|
||||
[DataContract]
|
||||
class OffloadDecision : PartitionUpdateEvent
|
||||
{
|
||||
[DataMember]
|
||||
public DateTime Timestamp { get; set; }
|
||||
|
||||
[IgnoreDataMember]
|
||||
public uint DestinationPartitionId { get; set; }
|
||||
|
||||
[IgnoreDataMember]
|
||||
public List<TaskMessage> OffloadedActivities { get; set; }
|
||||
|
||||
public static string GetWorkItemId(uint partition, DateTime timestamp) => $"{partition:D2}-O{timestamp:o}";
|
||||
|
||||
[IgnoreDataMember]
|
||||
public override EventId EventId => EventId.MakePartitionInternalEventId(GetWorkItemId(this.PartitionId, this.Timestamp));
|
||||
|
||||
public override void DetermineEffects(EffectTracker effects)
|
||||
{
|
||||
// start processing on activities, which makes the decision,
|
||||
// and if offloading, fills in the fields, and adds the outbox to the effects
|
||||
effects.Add(TrackedObjectKey.Activities);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
[DataContract]
|
||||
class PurgeBatchIssued : PartitionUpdateEvent
|
||||
{
|
||||
[DataMember]
|
||||
public string QueryEventId { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public int BatchNumber { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public List<string> InstanceIds { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public InstanceQuery InstanceQuery { get; set; }
|
||||
|
||||
[IgnoreDataMember]
|
||||
public TaskCompletionSource<object> WhenProcessed { get; set; }
|
||||
|
||||
[IgnoreDataMember]
|
||||
public override EventId EventId => EventId.MakePartitionInternalEventId(this.QueryEventId);
|
||||
|
||||
[IgnoreDataMember]
|
||||
public List<string> Purged { get; set; }
|
||||
|
||||
protected override void ExtraTraceInformation(StringBuilder s)
|
||||
{
|
||||
s.Append(" batchNumber=");
|
||||
s.Append(this.BatchNumber);
|
||||
s.Append(" count=");
|
||||
s.Append(this.InstanceIds.Count);
|
||||
}
|
||||
|
||||
public override void DetermineEffects(EffectTracker effects)
|
||||
{
|
||||
// the last-added effects are processed first
|
||||
// so they can set the Purged list to contain only the instance ids that are actually purged
|
||||
|
||||
effects.Add(TrackedObjectKey.Queries);
|
||||
effects.Add(TrackedObjectKey.Sessions);
|
||||
|
||||
this.Purged = new List<string>();
|
||||
foreach (string instanceId in this.InstanceIds)
|
||||
{
|
||||
effects.Add(TrackedObjectKey.Instance(instanceId));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System.Runtime.Serialization;
|
||||
|
||||
[DataContract]
|
||||
class SendConfirmed : PartitionUpdateEvent
|
||||
{
|
||||
[DataMember]
|
||||
public long Position { get; set; }
|
||||
|
||||
[IgnoreDataMember]
|
||||
public string WorkItemId => $"{this.PartitionId:D2}-C{this.Position:D10}";
|
||||
|
||||
[IgnoreDataMember]
|
||||
public override EventId EventId => EventId.MakePartitionInternalEventId(this.WorkItemId);
|
||||
|
||||
public override void DetermineEffects(EffectTracker effects)
|
||||
{
|
||||
effects.Add(TrackedObjectKey.Outbox);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.Serialization;
|
||||
using DurableTask.Core;
|
||||
|
||||
[DataContract]
|
||||
class TimerFired : PartitionUpdateEvent
|
||||
{
|
||||
[DataMember]
|
||||
public long TimerId { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public DateTime Due { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public TaskMessage TaskMessage { get; set; }
|
||||
|
||||
[IgnoreDataMember]
|
||||
public string WorkItemId => $"{this.PartitionId:D2}-T{this.TimerId}";
|
||||
|
||||
[IgnoreDataMember]
|
||||
public override EventId EventId => EventId.MakePartitionInternalEventId(this.WorkItemId);
|
||||
|
||||
[IgnoreDataMember]
|
||||
public override IEnumerable<TaskMessage> TracedTaskMessages { get { yield return this.TaskMessage; } }
|
||||
|
||||
public override void DetermineEffects(EffectTracker effects)
|
||||
{
|
||||
effects.Add(TrackedObjectKey.Sessions);
|
||||
effects.Add(TrackedObjectKey.Timers);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using System.Runtime.Serialization;
|
||||
|
||||
/// <summary>
|
||||
/// An event that is processed by a partition
|
||||
/// </summary>
|
||||
[DataContract]
|
||||
abstract class PartitionEvent : Event
|
||||
{
|
||||
[DataMember]
|
||||
public uint PartitionId { get; set; }
|
||||
|
||||
[IgnoreDataMember]
|
||||
public ArraySegment<byte> Serialized;
|
||||
|
||||
/// <summary>
|
||||
/// For events coming from the input queue, the next input queue position after this event. For internal events, zero.
|
||||
/// </summary>
|
||||
[DataMember]
|
||||
public long NextInputQueuePosition { get; set; }
|
||||
|
||||
[IgnoreDataMember]
|
||||
public double ReceivedTimestamp { get; set; }
|
||||
|
||||
[IgnoreDataMember]
|
||||
public double ReadyToSendTimestamp { get; set; }
|
||||
|
||||
[IgnoreDataMember]
|
||||
public double SentTimestamp { get; set; }
|
||||
|
||||
[IgnoreDataMember]
|
||||
public double IssuedTimestamp { get; set; }
|
||||
|
||||
// make a copy of an event so we run it through the pipeline a second time
|
||||
public PartitionEvent Clone()
|
||||
{
|
||||
var evt = (PartitionEvent)this.MemberwiseClone();
|
||||
|
||||
// clear all the non-data fields
|
||||
evt.DurabilityListeners.Clear();
|
||||
evt.Serialized = default;
|
||||
evt.NextInputQueuePosition = 0;
|
||||
|
||||
// clear the timestamps that will be overwritten
|
||||
evt.ReadyToSendTimestamp = 0;
|
||||
evt.SentTimestamp = 0;
|
||||
evt.IssuedTimestamp = 0;
|
||||
|
||||
return evt;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Threading.Tasks;
|
||||
using DurableTask.Core;
|
||||
|
||||
[DataContract]
|
||||
abstract class PartitionQueryEvent : PartitionEvent
|
||||
{
|
||||
public abstract InstanceQuery InstanceQuery { get; }
|
||||
|
||||
/// <summary>
|
||||
/// The continuation for the query operation.
|
||||
/// </summary>
|
||||
/// <param name="result">The tracked objects returned by this query</param>
|
||||
/// <param name="partition">The partition</param>
|
||||
public abstract Task OnQueryCompleteAsync(IAsyncEnumerable<OrchestrationState> result, Partition partition);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,89 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Text;
|
||||
|
||||
[DataContract]
|
||||
abstract class PartitionReadEvent : PartitionEvent
|
||||
{
|
||||
/// <summary>
|
||||
/// The target of the read operation.
|
||||
/// </summary>
|
||||
[IgnoreDataMember]
|
||||
public abstract TrackedObjectKey ReadTarget { get; }
|
||||
|
||||
/// <summary>
|
||||
/// A secondary target, to be prefetched also, before executing the read
|
||||
/// </summary>
|
||||
[IgnoreDataMember]
|
||||
public virtual TrackedObjectKey? Prefetch => null;
|
||||
|
||||
/// <summary>
|
||||
/// Optionally, some extra action to perform before issuing the read
|
||||
/// </summary>
|
||||
public virtual void OnReadIssued(Partition partition) { }
|
||||
|
||||
/// <summary>
|
||||
/// The continuation for the read operation.
|
||||
/// </summary>
|
||||
/// <param name="target">The current value of the tracked object for this key, or null if not present</param>
|
||||
/// <param name="partition">The partition</param>
|
||||
public abstract void OnReadComplete(TrackedObject target, Partition partition);
|
||||
|
||||
|
||||
#region prefetch state machine
|
||||
|
||||
[IgnoreDataMember]
|
||||
bool prefetchLoaded;
|
||||
|
||||
[IgnoreDataMember]
|
||||
bool targetLoaded;
|
||||
|
||||
[IgnoreDataMember]
|
||||
TrackedObject target;
|
||||
|
||||
public void Deliver(TrackedObjectKey key, TrackedObject trackedObject, out bool isReady)
|
||||
{
|
||||
if (!this.Prefetch.HasValue)
|
||||
{
|
||||
this.prefetchLoaded = true;
|
||||
this.targetLoaded = true;
|
||||
this.target = trackedObject;
|
||||
}
|
||||
else if (key.Equals(this.Prefetch.Value))
|
||||
{
|
||||
this.prefetchLoaded = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
this.targetLoaded = true;
|
||||
this.target = trackedObject;
|
||||
}
|
||||
|
||||
isReady = (this.prefetchLoaded && this.targetLoaded);
|
||||
}
|
||||
|
||||
public void Fire(Partition partition)
|
||||
{
|
||||
this.OnReadComplete(this.target, partition);
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Text;
|
||||
using DurableTask.Core;
|
||||
|
||||
[DataContract]
|
||||
abstract class PartitionUpdateEvent : PartitionEvent
|
||||
{
|
||||
/// <summary>
|
||||
/// The position of the next event after this one. For read-only events, zero.
|
||||
/// </summary>
|
||||
/// <remarks>We do not persist this in the log since it is implicit, nor transmit this in packets since it has only local meaning.</remarks>
|
||||
[IgnoreDataMember]
|
||||
public long NextCommitLogPosition { get; set; }
|
||||
|
||||
[IgnoreDataMember]
|
||||
public OutboxState.Batch OutboxBatch { get; set; }
|
||||
|
||||
[IgnoreDataMember]
|
||||
public virtual IEnumerable<TaskMessage> TracedTaskMessages => PartitionUpdateEvent.noTaskMessages;
|
||||
|
||||
static readonly IEnumerable<TaskMessage> noTaskMessages = Enumerable.Empty<TaskMessage>();
|
||||
|
||||
public abstract void DetermineEffects(EffectTracker effects);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Text;
|
||||
using DurableTask.Core;
|
||||
|
||||
class ActivityWorkItem : TaskActivityWorkItem
|
||||
{
|
||||
public Partition Partition { get; set; }
|
||||
|
||||
// the partition for the orchestration that issued this activity
|
||||
public uint OriginPartition { get; set; }
|
||||
|
||||
// a partition-local identifier for this activity (is a sequence number generated by ActivitiesState)
|
||||
public long ActivityId { get; set; }
|
||||
|
||||
public ActivityWorkItem(Partition partition, long activityId, TaskMessage message)
|
||||
{
|
||||
this.Partition = partition;
|
||||
this.OriginPartition = partition.PartitionFunction(message.OrchestrationInstance.InstanceId);
|
||||
this.ActivityId = activityId;
|
||||
this.Id = activityId.ToString();
|
||||
this.LockedUntilUtc = DateTime.MaxValue; // this backend does not require workitem lock renewals
|
||||
this.TaskMessage = message;
|
||||
}
|
||||
|
||||
public string WorkItemId => $"A{this.ActivityId}";
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,445 @@
|
|||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using DurableTask.Core;
|
||||
using DurableTask.Core.History;
|
||||
|
||||
class Client : TransportAbstraction.IClient
|
||||
{
|
||||
readonly NetheriteOrchestrationService host;
|
||||
readonly CancellationToken shutdownToken;
|
||||
readonly ClientTraceHelper traceHelper;
|
||||
readonly string account;
|
||||
readonly Guid taskHubGuid;
|
||||
|
||||
static readonly TimeSpan DefaultTimeout = TimeSpan.FromMinutes(5);
|
||||
|
||||
public Guid ClientId { get; private set; }
|
||||
TransportAbstraction.ISender BatchSender { get; set; }
|
||||
|
||||
long SequenceNumber; // for numbering requests that enter on this client
|
||||
|
||||
readonly BatchTimer<PendingRequest> ResponseTimeouts;
|
||||
readonly ConcurrentDictionary<long, PendingRequest> ResponseWaiters;
|
||||
readonly Dictionary<string, MemoryStream> Fragments;
|
||||
|
||||
public static string GetShortId(Guid clientId) => clientId.ToString("N").Substring(0, 7);
|
||||
|
||||
public Client(NetheriteOrchestrationService host, Guid clientId, Guid taskHubGuid, TransportAbstraction.ISender batchSender, CancellationToken shutdownToken)
|
||||
{
|
||||
this.host = host;
|
||||
this.ClientId = clientId;
|
||||
this.taskHubGuid = taskHubGuid;
|
||||
this.traceHelper = new ClientTraceHelper(host.Logger, host.Settings.LogLevelLimit, host.StorageAccountName, host.Settings.HubName, this.ClientId);
|
||||
this.account = host.StorageAccountName;
|
||||
this.BatchSender = batchSender;
|
||||
this.shutdownToken = shutdownToken;
|
||||
this.ResponseTimeouts = new BatchTimer<PendingRequest>(this.shutdownToken, this.Timeout, this.traceHelper.TraceTimerProgress);
|
||||
this.ResponseWaiters = new ConcurrentDictionary<long, PendingRequest>();
|
||||
this.Fragments = new Dictionary<string, MemoryStream>();
|
||||
this.ResponseTimeouts.Start("ClientTimer");
|
||||
|
||||
this.traceHelper.TraceProgress("Started");
|
||||
}
|
||||
|
||||
public Task StopAsync()
|
||||
{
|
||||
this.traceHelper.TraceProgress("Stopped");
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public void ReportTransportError(string message, Exception e)
|
||||
{
|
||||
this.traceHelper.TraceError("ReportTransportError", message, e);
|
||||
}
|
||||
|
||||
public void Process(ClientEvent clientEvent)
|
||||
{
|
||||
if (!(clientEvent is ClientEventFragment fragment))
|
||||
{
|
||||
this.ProcessInternal(clientEvent);
|
||||
}
|
||||
else
|
||||
{
|
||||
var originalEventString = fragment.OriginalEventId.ToString();
|
||||
|
||||
if (!fragment.IsLast)
|
||||
{
|
||||
if (!this.Fragments.TryGetValue(originalEventString, out var stream))
|
||||
{
|
||||
this.Fragments[originalEventString] = stream = new MemoryStream();
|
||||
}
|
||||
stream.Write(fragment.Bytes, 0, fragment.Bytes.Length);
|
||||
}
|
||||
else
|
||||
{
|
||||
var reassembledEvent = FragmentationAndReassembly.Reassemble<ClientEvent>(this.Fragments[originalEventString], fragment);
|
||||
this.Fragments.Remove(fragment.EventIdString);
|
||||
|
||||
this.ProcessInternal(reassembledEvent);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void ProcessInternal(ClientEvent clientEvent)
|
||||
{
|
||||
this.traceHelper.TraceReceive(clientEvent);
|
||||
if (this.ResponseWaiters.TryRemove(clientEvent.RequestId, out var waiter))
|
||||
{
|
||||
waiter.Respond(clientEvent);
|
||||
}
|
||||
}
|
||||
|
||||
public void Send(PartitionEvent partitionEvent)
|
||||
{
|
||||
this.traceHelper.TraceSend(partitionEvent);
|
||||
this.BatchSender.Submit(partitionEvent);
|
||||
}
|
||||
|
||||
void Timeout(List<PendingRequest> pendingRequests)
|
||||
{
|
||||
Parallel.ForEach(pendingRequests, pendingRequest => pendingRequest.TryTimeout());
|
||||
}
|
||||
|
||||
// we align timeouts into buckets so we can process timeout storms more efficiently
|
||||
const long ticksPerBucket = 2 * TimeSpan.TicksPerSecond;
|
||||
DateTime GetTimeoutBucket(TimeSpan timeout) => new DateTime((((DateTime.UtcNow + timeout).Ticks / ticksPerBucket) * ticksPerBucket), DateTimeKind.Utc);
|
||||
|
||||
Task<ClientEvent> PerformRequestWithTimeoutAndCancellation(CancellationToken token, IClientRequestEvent request, bool doneWhenSent)
|
||||
{
|
||||
var partitionEvent = (PartitionEvent)request;
|
||||
int timeoutId = this.ResponseTimeouts.GetFreshId();
|
||||
var pendingRequest = new PendingRequest(request.RequestId, request.EventId, partitionEvent.PartitionId, this, request.TimeoutUtc, timeoutId);
|
||||
this.ResponseWaiters.TryAdd(request.RequestId, pendingRequest);
|
||||
this.ResponseTimeouts.Schedule(request.TimeoutUtc, pendingRequest, timeoutId);
|
||||
|
||||
if (doneWhenSent)
|
||||
{
|
||||
DurabilityListeners.Register((Event)request, pendingRequest);
|
||||
}
|
||||
|
||||
this.Send(partitionEvent);
|
||||
|
||||
return pendingRequest.Task;
|
||||
}
|
||||
|
||||
internal class PendingRequest : TransportAbstraction.IDurabilityOrExceptionListener
|
||||
{
|
||||
readonly long requestId;
|
||||
readonly EventId eventId;
|
||||
readonly uint partitionId;
|
||||
readonly Client client;
|
||||
readonly (DateTime due, int id) timeoutKey;
|
||||
readonly TaskCompletionSource<ClientEvent> continuation;
|
||||
|
||||
static readonly TimeoutException timeoutException = new TimeoutException("Client request timed out.");
|
||||
|
||||
public Task<ClientEvent> Task => this.continuation.Task;
|
||||
public (DateTime, int) TimeoutKey => this.timeoutKey;
|
||||
|
||||
public string RequestId => $"{Client.GetShortId(this.client.ClientId)}-{this.requestId}"; // matches EventId
|
||||
|
||||
public PendingRequest(long requestId, EventId eventId, uint partitionId, Client client, DateTime due, int timeoutId)
|
||||
{
|
||||
this.requestId = requestId;
|
||||
this.partitionId = partitionId;
|
||||
this.eventId = eventId;
|
||||
this.client = client;
|
||||
this.timeoutKey = (due, timeoutId);
|
||||
this.continuation = new TaskCompletionSource<ClientEvent>(TaskContinuationOptions.ExecuteSynchronously);
|
||||
}
|
||||
|
||||
public void Respond(ClientEvent evt)
|
||||
{
|
||||
this.client.ResponseTimeouts.TryCancel(this.timeoutKey);
|
||||
this.continuation.TrySetResult(evt);
|
||||
}
|
||||
|
||||
void TransportAbstraction.IDurabilityListener.ConfirmDurable(Event evt)
|
||||
{
|
||||
if (this.client.ResponseWaiters.TryRemove(this.requestId, out var _))
|
||||
{
|
||||
this.client.ResponseTimeouts.TryCancel(this.timeoutKey);
|
||||
this.continuation.TrySetResult(null); // task finishes when the send has been confirmed, no result is returned
|
||||
}
|
||||
}
|
||||
|
||||
void TransportAbstraction.IDurabilityOrExceptionListener.ReportException(Event evt, Exception e)
|
||||
{
|
||||
if (this.client.ResponseWaiters.TryRemove(this.requestId, out var _))
|
||||
{
|
||||
this.client.ResponseTimeouts.TryCancel(this.timeoutKey);
|
||||
this.continuation.TrySetException(e); // task finishes with exception
|
||||
}
|
||||
}
|
||||
|
||||
public void TryTimeout()
|
||||
{
|
||||
this.client.traceHelper.TraceTimerProgress($"firing ({this.timeoutKey.due:o},{this.timeoutKey.id})");
|
||||
if (this.client.ResponseWaiters.TryRemove(this.requestId, out var pendingRequest))
|
||||
{
|
||||
this.client.traceHelper.TraceRequestTimeout(pendingRequest.eventId, pendingRequest.partitionId);
|
||||
this.continuation.TrySetException(timeoutException);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/******************************/
|
||||
// orchestration client methods
|
||||
/******************************/
|
||||
|
||||
public async Task CreateTaskOrchestrationAsync(uint partitionId, TaskMessage creationMessage, OrchestrationStatus[] dedupeStatuses)
|
||||
{
|
||||
ExecutionStartedEvent executionStartedEvent = creationMessage.Event as ExecutionStartedEvent;
|
||||
if (executionStartedEvent == null)
|
||||
{
|
||||
throw new ArgumentException($"Only {nameof(EventType.ExecutionStarted)} messages are supported.", nameof(creationMessage));
|
||||
}
|
||||
|
||||
var request = new CreationRequestReceived()
|
||||
{
|
||||
PartitionId = partitionId,
|
||||
ClientId = this.ClientId,
|
||||
RequestId = Interlocked.Increment(ref this.SequenceNumber),
|
||||
TaskMessage = creationMessage,
|
||||
DedupeStatuses = dedupeStatuses,
|
||||
Timestamp = DateTime.UtcNow,
|
||||
TimeoutUtc = this.GetTimeoutBucket(DefaultTimeout),
|
||||
};
|
||||
|
||||
var response = await this.PerformRequestWithTimeoutAndCancellation(this.shutdownToken, request, false).ConfigureAwait(false);
|
||||
var creationResponseReceived = (CreationResponseReceived)response;
|
||||
if (!creationResponseReceived.Succeeded)
|
||||
{
|
||||
// An instance in this state already exists.
|
||||
if (this.host.Settings.ThrowExceptionOnInvalidDedupeStatus)
|
||||
{
|
||||
throw new InvalidOperationException($"An Orchestration instance with the status {creationResponseReceived.ExistingInstanceOrchestrationStatus} already exists.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public Task SendTaskOrchestrationMessageBatchAsync(uint partitionId, IEnumerable<TaskMessage> messages)
|
||||
{
|
||||
var request = new ClientTaskMessagesReceived()
|
||||
{
|
||||
PartitionId = partitionId,
|
||||
ClientId = this.ClientId,
|
||||
RequestId = Interlocked.Increment(ref this.SequenceNumber),
|
||||
TaskMessages = messages.ToArray(),
|
||||
TimeoutUtc = this.GetTimeoutBucket(DefaultTimeout),
|
||||
};
|
||||
|
||||
return this.PerformRequestWithTimeoutAndCancellation(this.shutdownToken, request, true);
|
||||
}
|
||||
|
||||
public async Task<OrchestrationState> WaitForOrchestrationAsync(
|
||||
uint partitionId,
|
||||
string instanceId,
|
||||
string executionId,
|
||||
TimeSpan timeout,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(instanceId))
|
||||
{
|
||||
throw new ArgumentException(nameof(instanceId));
|
||||
}
|
||||
|
||||
var request = new WaitRequestReceived()
|
||||
{
|
||||
PartitionId = partitionId,
|
||||
ClientId = this.ClientId,
|
||||
RequestId = Interlocked.Increment(ref this.SequenceNumber),
|
||||
InstanceId = instanceId,
|
||||
ExecutionId = executionId,
|
||||
TimeoutUtc = this.GetTimeoutBucket(timeout),
|
||||
};
|
||||
|
||||
var response = await this.PerformRequestWithTimeoutAndCancellation(cancellationToken, request, false).ConfigureAwait(false);
|
||||
return ((WaitResponseReceived)response)?.OrchestrationState;
|
||||
}
|
||||
|
||||
public async Task<OrchestrationState> GetOrchestrationStateAsync(uint partitionId, string instanceId, bool fetchInput = true, bool fetchOutput = true)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(instanceId))
|
||||
{
|
||||
throw new ArgumentException(nameof(instanceId));
|
||||
}
|
||||
|
||||
var request = new StateRequestReceived()
|
||||
{
|
||||
PartitionId = partitionId,
|
||||
ClientId = this.ClientId,
|
||||
RequestId = Interlocked.Increment(ref this.SequenceNumber),
|
||||
InstanceId = instanceId,
|
||||
IncludeInput = fetchInput,
|
||||
IncludeOutput = fetchOutput,
|
||||
TimeoutUtc = this.GetTimeoutBucket(DefaultTimeout),
|
||||
};
|
||||
|
||||
var response = await this.PerformRequestWithTimeoutAndCancellation(this.shutdownToken, request, false).ConfigureAwait(false);
|
||||
return ((StateResponseReceived)response)?.OrchestrationState;
|
||||
}
|
||||
|
||||
public async Task<(string executionId, IList<HistoryEvent> history)> GetOrchestrationHistoryAsync(uint partitionId, string instanceId)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(instanceId))
|
||||
{
|
||||
throw new ArgumentException(nameof(instanceId));
|
||||
}
|
||||
|
||||
var request = new HistoryRequestReceived()
|
||||
{
|
||||
PartitionId = partitionId,
|
||||
ClientId = this.ClientId,
|
||||
RequestId = Interlocked.Increment(ref this.SequenceNumber),
|
||||
InstanceId = instanceId,
|
||||
TimeoutUtc = this.GetTimeoutBucket(DefaultTimeout),
|
||||
};
|
||||
|
||||
var response = (HistoryResponseReceived)await this.PerformRequestWithTimeoutAndCancellation(this.shutdownToken, request, false).ConfigureAwait(false);
|
||||
return (response?.ExecutionId, response?.History);
|
||||
}
|
||||
|
||||
public Task<IList<OrchestrationState>> GetOrchestrationStateAsync(CancellationToken cancellationToken)
|
||||
=> this.RunPartitionQueries<InstanceQueryReceived, QueryResponseReceived, IList<OrchestrationState>>(
|
||||
partitionId => new InstanceQueryReceived() {
|
||||
PartitionId = partitionId,
|
||||
ClientId = this.ClientId,
|
||||
RequestId = Interlocked.Increment(ref this.SequenceNumber),
|
||||
TimeoutUtc = this.GetTimeoutBucket(DefaultTimeout),
|
||||
InstanceQuery = new InstanceQuery(),
|
||||
},
|
||||
(IEnumerable<QueryResponseReceived> responses) => responses.SelectMany(response => response.OrchestrationStates).ToList(),
|
||||
cancellationToken);
|
||||
|
||||
public Task<IList<OrchestrationState>> GetOrchestrationStateAsync(DateTime? createdTimeFrom, DateTime? createdTimeTo,
|
||||
IEnumerable<OrchestrationStatus> runtimeStatus, string instanceIdPrefix, CancellationToken cancellationToken = default)
|
||||
=> this.RunPartitionQueries<InstanceQueryReceived,QueryResponseReceived,IList<OrchestrationState>>(
|
||||
partitionId => new InstanceQueryReceived() {
|
||||
PartitionId = partitionId,
|
||||
ClientId = this.ClientId,
|
||||
RequestId = Interlocked.Increment(ref this.SequenceNumber),
|
||||
TimeoutUtc = this.GetTimeoutBucket(DefaultTimeout),
|
||||
InstanceQuery = new InstanceQuery(
|
||||
runtimeStatus?.ToArray(),
|
||||
createdTimeFrom?.ToUniversalTime(),
|
||||
createdTimeTo?.ToUniversalTime(),
|
||||
instanceIdPrefix,
|
||||
fetchInput: true),
|
||||
},
|
||||
(IEnumerable<QueryResponseReceived> responses) => responses.SelectMany(response => response.OrchestrationStates).ToList(),
|
||||
cancellationToken);
|
||||
|
||||
public Task<int> PurgeInstanceHistoryAsync(DateTime? createdTimeFrom, DateTime? createdTimeTo, IEnumerable<OrchestrationStatus> runtimeStatus, CancellationToken cancellationToken = default)
|
||||
=> this.RunPartitionQueries(
|
||||
partitionId => new PurgeRequestReceived()
|
||||
{
|
||||
PartitionId = partitionId,
|
||||
ClientId = this.ClientId,
|
||||
RequestId = Interlocked.Increment(ref this.SequenceNumber),
|
||||
TimeoutUtc = this.GetTimeoutBucket(DefaultTimeout),
|
||||
InstanceQuery = new InstanceQuery(
|
||||
runtimeStatus?.ToArray(),
|
||||
createdTimeFrom?.ToUniversalTime(),
|
||||
createdTimeTo?.ToUniversalTime(),
|
||||
null,
|
||||
fetchInput: false)
|
||||
{ PrefetchHistory = true },
|
||||
},
|
||||
(IEnumerable<PurgeResponseReceived> responses) => responses.Sum(response => response.NumberInstancesPurged),
|
||||
cancellationToken);
|
||||
|
||||
public Task<InstanceQueryResult> QueryOrchestrationStatesAsync(InstanceQuery instanceQuery, int pageSize, string continuationToken, CancellationToken cancellationToken)
|
||||
=> this.RunPartitionQueries(
|
||||
partitionId => new InstanceQueryReceived()
|
||||
{
|
||||
PartitionId = partitionId,
|
||||
ClientId = this.ClientId,
|
||||
RequestId = Interlocked.Increment(ref this.SequenceNumber),
|
||||
TimeoutUtc = this.GetTimeoutBucket(DefaultTimeout),
|
||||
InstanceQuery = instanceQuery,
|
||||
},
|
||||
(IEnumerable<QueryResponseReceived> responses) => new InstanceQueryResult()
|
||||
{
|
||||
Instances = responses.SelectMany(response => response.OrchestrationStates),
|
||||
ContinuationToken = null,
|
||||
},
|
||||
cancellationToken);
|
||||
|
||||
async Task<TResult> RunPartitionQueries<TRequest,TResponse,TResult>(
|
||||
Func<uint, TRequest> requestCreator,
|
||||
Func<IEnumerable<TResponse>,TResult> responseAggregator,
|
||||
CancellationToken cancellationToken)
|
||||
where TRequest: IClientRequestEvent
|
||||
{
|
||||
IEnumerable<Task<ClientEvent>> launchQueries()
|
||||
{
|
||||
for (uint partitionId = 0; partitionId < this.host.NumberPartitions; ++partitionId)
|
||||
{
|
||||
yield return this.PerformRequestWithTimeoutAndCancellation(cancellationToken, requestCreator(partitionId), false);
|
||||
}
|
||||
}
|
||||
|
||||
return responseAggregator((await Task.WhenAll(launchQueries()).ConfigureAwait(false)).Cast<TResponse>());
|
||||
}
|
||||
|
||||
public Task ForceTerminateTaskOrchestrationAsync(uint partitionId, string instanceId, string message)
|
||||
{
|
||||
var taskMessages = new[] { new TaskMessage
|
||||
{
|
||||
OrchestrationInstance = new OrchestrationInstance { InstanceId = instanceId },
|
||||
Event = new ExecutionTerminatedEvent(-1, message)
|
||||
} };
|
||||
|
||||
var request = new ClientTaskMessagesReceived()
|
||||
{
|
||||
PartitionId = partitionId,
|
||||
ClientId = this.ClientId,
|
||||
RequestId = Interlocked.Increment(ref this.SequenceNumber),
|
||||
TaskMessages = taskMessages,
|
||||
TimeoutUtc = this.GetTimeoutBucket(DefaultTimeout),
|
||||
};
|
||||
|
||||
return this.PerformRequestWithTimeoutAndCancellation(CancellationToken.None, request, true);
|
||||
}
|
||||
|
||||
public async Task<int> DeleteAllDataForOrchestrationInstance(uint partitionId, string instanceId)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(instanceId))
|
||||
{
|
||||
throw new ArgumentException(nameof(instanceId));
|
||||
}
|
||||
|
||||
var request = new DeletionRequestReceived()
|
||||
{
|
||||
PartitionId = partitionId,
|
||||
ClientId = this.ClientId,
|
||||
RequestId = Interlocked.Increment(ref this.SequenceNumber),
|
||||
InstanceId = instanceId,
|
||||
TimeoutUtc = this.GetTimeoutBucket(DefaultTimeout),
|
||||
};
|
||||
|
||||
var response = await this.PerformRequestWithTimeoutAndCancellation(this.shutdownToken, request, false).ConfigureAwait(false);
|
||||
return ((DeletionResponseReceived)response).NumberInstancesDeleted;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,130 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using Microsoft.Extensions.Logging;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
|
||||
class ClientTraceHelper
|
||||
{
|
||||
readonly ILogger logger;
|
||||
readonly string account;
|
||||
readonly string taskHub;
|
||||
readonly Guid clientId;
|
||||
readonly LogLevel logLevelLimit;
|
||||
readonly string tracePrefix;
|
||||
|
||||
public ClientTraceHelper(ILogger logger, LogLevel logLevelLimit, string storageAccountName, string taskHubName, Guid clientId)
|
||||
{
|
||||
this.logger = logger;
|
||||
this.account = storageAccountName;
|
||||
this.taskHub = taskHubName;
|
||||
this.clientId = clientId;
|
||||
this.logLevelLimit = logLevelLimit;
|
||||
this.tracePrefix = $"Client.{Client.GetShortId(clientId)}";
|
||||
}
|
||||
|
||||
public void TraceProgress(string details)
|
||||
{
|
||||
if (this.logLevelLimit <= LogLevel.Information)
|
||||
{
|
||||
if (this.logger.IsEnabled(LogLevel.Information))
|
||||
{
|
||||
this.logger.LogInformation("{client} {details}", this.tracePrefix, details);
|
||||
}
|
||||
if (EtwSource.Log.IsEnabled())
|
||||
{
|
||||
EtwSource.Log.ClientProgress(this.account, this.taskHub, this.clientId, details, TraceUtils.ExtensionVersion);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void TraceError(string context, string message, Exception exception)
|
||||
{
|
||||
if (this.logLevelLimit <= LogLevel.Error)
|
||||
{
|
||||
if (this.logger.IsEnabled(LogLevel.Error))
|
||||
{
|
||||
this.logger.LogError("{client} !!! {message}: {exception}", this.tracePrefix, message, exception);
|
||||
}
|
||||
if (EtwSource.Log.IsEnabled())
|
||||
{
|
||||
EtwSource.Log.ClientError(this.account, this.taskHub, this.clientId, context, message, exception.ToString(), TraceUtils.ExtensionVersion);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void TraceTimerProgress(string details)
|
||||
{
|
||||
if (this.logLevelLimit <= LogLevel.Debug)
|
||||
{
|
||||
if (this.logger.IsEnabled(LogLevel.Debug))
|
||||
{
|
||||
this.logger.LogInformation("{client} {details}", this.tracePrefix, details);
|
||||
}
|
||||
if (EtwSource.Log.IsEnabled())
|
||||
{
|
||||
EtwSource.Log.ClientTimerProgress(this.account, this.taskHub, this.clientId, details, TraceUtils.ExtensionVersion);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void TraceRequestTimeout(EventId eventId, uint partitionId)
|
||||
{
|
||||
if (this.logLevelLimit <= LogLevel.Warning)
|
||||
{
|
||||
if (this.logger.IsEnabled(LogLevel.Warning))
|
||||
{
|
||||
this.logger.LogWarning("{client} Request {eventId} for partition {partitionId:D2} timed out", this.tracePrefix, eventId, partitionId);
|
||||
}
|
||||
if (EtwSource.Log.IsEnabled())
|
||||
{
|
||||
EtwSource.Log.ClientRequestTimeout(this.account, this.taskHub, this.clientId, eventId.ToString(), (int) partitionId, TraceUtils.ExtensionVersion);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void TraceSend(Event @event)
|
||||
{
|
||||
if (this.logLevelLimit <= LogLevel.Debug)
|
||||
{
|
||||
if (this.logger.IsEnabled(LogLevel.Debug))
|
||||
{
|
||||
this.logger.LogDebug("{client} Sending event {eventId}: {event}", this.tracePrefix, @event.EventIdString, @event);
|
||||
}
|
||||
if (EtwSource.Log.IsEnabled())
|
||||
{
|
||||
EtwSource.Log.ClientEventSent(this.account, this.taskHub, this.clientId, @event.EventIdString, @event.ToString(), TraceUtils.ExtensionVersion);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void TraceReceive(Event @event)
|
||||
{
|
||||
if (this.logLevelLimit <= LogLevel.Debug)
|
||||
{
|
||||
if (this.logger.IsEnabled(LogLevel.Debug))
|
||||
{
|
||||
this.logger.LogDebug("{client} Processing event {eventId}: {event}", this.tracePrefix, @event.EventIdString, @event);
|
||||
}
|
||||
if (EtwSource.Log.IsEnabled())
|
||||
{
|
||||
EtwSource.Log.ClientEventReceived(this.account, this.taskHub, this.clientId, @event.EventIdString, @event.ToString(), TraceUtils.ExtensionVersion);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,370 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using System.Diagnostics.Tracing;
|
||||
|
||||
/// <summary>
|
||||
/// ETW Event Provider for the DurableTask.Netherite provider extension.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The ETW Provider ID for this event source is {b3b94da0-1edd-53a7-435e-53129d278be4}.
|
||||
/// We list all events from the various layers (transport, storage) in this single file; however,
|
||||
/// we do have separate helper classes for each component.
|
||||
/// </remarks>
|
||||
[EventSource(Name = "DurableTask-Netherite")]
|
||||
class EtwSource : EventSource
|
||||
{
|
||||
/// <summary>
|
||||
/// Singleton instance used for writing events.
|
||||
/// </summary>
|
||||
public static readonly EtwSource Log = new EtwSource();
|
||||
|
||||
// we should always check if verbose is enabled before doing extensive string formatting for a verbose event
|
||||
public bool IsVerboseEnabled => this.IsEnabled(EventLevel.Verbose, EventKeywords.None);
|
||||
|
||||
// ----- orchestration service lifecycle
|
||||
|
||||
// we are grouping all events by this instance of NetheriteOrchestrationService using a single activity id
|
||||
// and since there is only one of these per machine, we can save its id in this static field.
|
||||
static Guid serviceInstanceId;
|
||||
|
||||
[Event(200, Level = EventLevel.Informational, Opcode = EventOpcode.Start, Version = 1)]
|
||||
public void OrchestrationServiceCreated(Guid OrchestrationServiceInstanceId, string Account, string TaskHub, string WorkerName, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(OrchestrationServiceInstanceId);
|
||||
this.WriteEvent(200, OrchestrationServiceInstanceId, Account, TaskHub, WorkerName, ExtensionVersion);
|
||||
EtwSource.serviceInstanceId = OrchestrationServiceInstanceId;
|
||||
}
|
||||
|
||||
[Event(201, Level = EventLevel.Informational, Opcode = EventOpcode.Stop, Version = 1)]
|
||||
public void OrchestrationServiceStopped(Guid OrchestrationServiceInstanceId, string Account, string TaskHub, string WorkerName, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(OrchestrationServiceInstanceId);
|
||||
this.WriteEvent(201, OrchestrationServiceInstanceId, Account, TaskHub, WorkerName, ExtensionVersion);
|
||||
}
|
||||
|
||||
// ----- partition and client lifecycles
|
||||
|
||||
[Event(210, Level = EventLevel.Informational, Version = 1)]
|
||||
public void PartitionProgress(string Account, string TaskHub, int PartitionId, string Details, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(serviceInstanceId);
|
||||
this.WriteEvent(210, Account, TaskHub, PartitionId, Details, ExtensionVersion);
|
||||
}
|
||||
|
||||
[Event(211, Level = EventLevel.Warning, Version = 1)]
|
||||
public void PartitionWarning(string Account, string TaskHub, int PartitionId, string Context, bool TerminatesPartition, string Message, string Details, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(serviceInstanceId);
|
||||
this.WriteEvent(211, Account, TaskHub, PartitionId, Context, TerminatesPartition, Message, Details, ExtensionVersion);
|
||||
}
|
||||
|
||||
[Event(212, Level = EventLevel.Error, Version = 1)]
|
||||
public void PartitionError(string Account, string TaskHub, int PartitionId, string Context, bool TerminatesPartition, string Message, string Details, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(serviceInstanceId);
|
||||
this.WriteEvent(212, Account, TaskHub, PartitionId, Context, TerminatesPartition, Message, Details, ExtensionVersion);
|
||||
}
|
||||
|
||||
[Event(213, Level = EventLevel.Informational, Version = 1)]
|
||||
public void ClientProgress(string Account, string TaskHub, Guid ClientId, string Details, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(serviceInstanceId);
|
||||
this.WriteEvent(213, Account, TaskHub, ClientId, Details, ExtensionVersion);
|
||||
}
|
||||
|
||||
[Event(214, Level = EventLevel.Warning, Version = 1)]
|
||||
public void ClientWarning(string Account, string TaskHub, Guid ClientId, string Context, string Message, string Details, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(serviceInstanceId);
|
||||
this.WriteEvent(214, Account, TaskHub, ClientId, Context, Message, Details, ExtensionVersion);
|
||||
}
|
||||
|
||||
[Event(215, Level = EventLevel.Error, Version = 1)]
|
||||
public void ClientError(string Account, string TaskHub, Guid ClientId, string Context, string Message, string Details, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(serviceInstanceId);
|
||||
this.WriteEvent(215, Account, TaskHub, ClientId, Context, Message, Details, ExtensionVersion);
|
||||
}
|
||||
|
||||
[Event(216, Level = EventLevel.Verbose, Version = 1)]
|
||||
public void ClientTimerProgress(string Account, string TaskHub, Guid ClientId, string Details, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(serviceInstanceId);
|
||||
this.WriteEvent(216, Account, TaskHub, ClientId, Details, ExtensionVersion);
|
||||
}
|
||||
|
||||
[Event(217, Level = EventLevel.Warning, Version = 1)]
|
||||
public void ClientRequestTimeout(string Account, string TaskHub, Guid ClientId, string EventId, int PartitionId, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(serviceInstanceId);
|
||||
this.WriteEvent(217, Account, TaskHub, ClientId, EventId, PartitionId, ExtensionVersion);
|
||||
}
|
||||
|
||||
// ----- specific events relating to DurableTask concepts (TaskMessage, OrchestrationWorkItem, Instance)
|
||||
|
||||
[Event(220, Level = EventLevel.Verbose, Version = 1)]
|
||||
public void TaskMessageReceived(string Account, string TaskHub, int PartitionId, long CommitLogPosition, string EventType, int TaskEventId, string InstanceId, string ExecutionId, string MessageId, string SessionPosition, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(serviceInstanceId);
|
||||
this.WriteEvent(220, Account, TaskHub, PartitionId, CommitLogPosition, EventType, TaskEventId, InstanceId, ExecutionId, MessageId, SessionPosition, ExtensionVersion);
|
||||
}
|
||||
|
||||
[Event(221, Level = EventLevel.Verbose, Version = 1)]
|
||||
public void TaskMessageSent(string Account, string TaskHub, int PartitionId, string EventType, int TaskEventId, string InstanceId, string ExecutionId, string MessageId, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(serviceInstanceId);
|
||||
this.WriteEvent(221, Account, TaskHub, PartitionId, EventType, TaskEventId, InstanceId, ExecutionId, MessageId, ExtensionVersion);
|
||||
}
|
||||
|
||||
[Event(222, Level = EventLevel.Warning, Version = 1)]
|
||||
public void TaskMessageDiscarded(string Account, string TaskHub, int PartitionId, string Reason, string EventType, int TaskEventId, string InstanceId, string ExecutionId, string WorkItemId, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(serviceInstanceId);
|
||||
this.WriteEvent(222, Account, TaskHub, PartitionId, Reason, EventType, TaskEventId, InstanceId, ExecutionId, WorkItemId, ExtensionVersion);
|
||||
}
|
||||
|
||||
[Event(223, Level = EventLevel.Verbose, Version = 1)]
|
||||
public void OrchestrationWorkItemQueued(string Account, string TaskHub, int PartitionId, string ExecutionType, string InstanceId, string WorkItemId, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(serviceInstanceId);
|
||||
this.WriteEvent(223, Account, TaskHub, PartitionId, ExecutionType, InstanceId, WorkItemId, ExtensionVersion);
|
||||
}
|
||||
|
||||
[Event(224, Level = EventLevel.Warning, Version = 1)]
|
||||
public void OrchestrationWorkItemDiscarded(string Account, string TaskHub, int PartitionId, string InstanceId, string WorkItemId, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(serviceInstanceId);
|
||||
this.WriteEvent(224, Account, TaskHub, PartitionId, InstanceId, WorkItemId, ExtensionVersion);
|
||||
}
|
||||
|
||||
[Event(225, Level = EventLevel.Verbose, Version = 1)]
|
||||
public void InstanceUpdated(string Account, string TaskHub, int PartitionId, string InstanceId, string ExecutionId, string WorkItemId, int NewEventCount, int TotalEventCount, string NewEvents, string EventType, int Episode, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(serviceInstanceId);
|
||||
this.WriteEvent(225, Account, TaskHub, PartitionId, InstanceId, ExecutionId, WorkItemId, NewEventCount, TotalEventCount, NewEvents, EventType, Episode, ExtensionVersion);
|
||||
}
|
||||
|
||||
[Event(226, Level = EventLevel.Verbose, Version = 1)]
|
||||
public void InstanceStatusFetched(string Account, string TaskHub, int PartitionId, string InstanceId, string ExecutionId, string status, string EventId, double LatencyMs, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(serviceInstanceId);
|
||||
this.WriteEvent(226, Account, TaskHub, PartitionId, InstanceId, ExecutionId, status, EventId, LatencyMs, ExtensionVersion);
|
||||
}
|
||||
|
||||
[Event(227, Level = EventLevel.Verbose, Version = 1)]
|
||||
public void InstanceHistoryFetched(string Account, string TaskHub, int PartitionId, string InstanceId, string ExecutionId, int EventCount, int Episode, string EventId, double LatencyMs, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(serviceInstanceId);
|
||||
this.WriteEvent(227, Account, TaskHub, PartitionId, InstanceId, ExecutionId, EventCount, Episode, EventId, LatencyMs, ExtensionVersion);
|
||||
}
|
||||
|
||||
// ----- general event processing and statistics
|
||||
|
||||
[Event(240, Level = EventLevel.Informational, Version = 1)]
|
||||
public void PartitionEventProcessed(string Account, string TaskHub, int PartitionId, long CommitLogPosition, string MessageId, string EventInfo, long NextCommitLogPosition, long NextInputQueuePosition, double QueueLatencyMs, double FetchLatencyMs, double LatencyMs, bool IsReplaying, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(serviceInstanceId);
|
||||
this.WriteEvent(240, Account, TaskHub, PartitionId, CommitLogPosition, MessageId, EventInfo, NextCommitLogPosition, NextInputQueuePosition, QueueLatencyMs, FetchLatencyMs, LatencyMs, IsReplaying, ExtensionVersion);
|
||||
}
|
||||
|
||||
[Event(241, Level = EventLevel.Verbose, Version = 1)]
|
||||
public void PartitionEventDetail(string Account, string TaskHub, int PartitionId, long CommitLogPosition, string MessageId, string Details, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(serviceInstanceId);
|
||||
this.WriteEvent(241, Account, TaskHub, PartitionId, CommitLogPosition, MessageId, Details, ExtensionVersion);
|
||||
}
|
||||
|
||||
[Event(242, Level = EventLevel.Verbose, Version = 1)]
|
||||
public void ClientEventReceived(string Account, string TaskHub, Guid ClientId, string MessageId, string EventInfo, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(serviceInstanceId);
|
||||
this.WriteEvent(242, Account, TaskHub, ClientId, MessageId, EventInfo, ExtensionVersion);
|
||||
}
|
||||
|
||||
[Event(243, Level = EventLevel.Verbose, Version = 1)]
|
||||
public void ClientEventSent(string Account, string TaskHub, Guid ClientId, string MessageId, string EventInfo, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(serviceInstanceId);
|
||||
this.WriteEvent(243, Account, TaskHub, ClientId, MessageId, EventInfo, ExtensionVersion);
|
||||
}
|
||||
|
||||
[Event(244, Level = EventLevel.Warning, Version = 1)]
|
||||
public void PartitionOffloadDecision(string Account, string TaskHub, int PartitionId, long CommitLogPosition, string MessageId, int ReportedLocalLoad, int Pending, int Backlog, int Remotes, string ReportedRemoteLoad, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(serviceInstanceId);
|
||||
this.WriteEvent(244, Account, TaskHub, PartitionId, CommitLogPosition, MessageId, ReportedLocalLoad, Pending, Backlog, Remotes, ReportedRemoteLoad, ExtensionVersion);
|
||||
}
|
||||
|
||||
[Event(245, Level = EventLevel.Informational, Version = 1)]
|
||||
public void PartitionLoadPublished(string Account, string TaskHub, int PartitionId, int WorkItems, int Activities, int Timers, int Requests, int Outbox, string NextTimer, long ActivityLatencyMs, long WorkItemLatencyMs, string WorkerId, string LatencyTrend, double MissRate, long InputQueuePosition, long CommitLogPosition, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(serviceInstanceId);
|
||||
this.WriteEvent(245, Account, TaskHub, PartitionId, WorkItems, Activities, Timers, Requests, Outbox, NextTimer, ActivityLatencyMs, WorkItemLatencyMs, WorkerId, LatencyTrend, MissRate, InputQueuePosition, CommitLogPosition, ExtensionVersion);
|
||||
}
|
||||
|
||||
// ----- Faster Storage
|
||||
|
||||
[Event(250, Level = EventLevel.Informational, Version = 1)]
|
||||
public void FasterStoreCreated(string Account, string TaskHub, int PartitionId, long InputQueuePosition, long LatencyMs, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(serviceInstanceId);
|
||||
this.WriteEvent(250, Account, TaskHub, PartitionId, InputQueuePosition, LatencyMs, ExtensionVersion);
|
||||
}
|
||||
|
||||
[Event(251, Level = EventLevel.Informational, Version = 1)]
|
||||
public void FasterCheckpointStarted(string Account, string TaskHub, int PartitionId, Guid CheckpointId, string Reason, string StoreStats, long CommitLogPosition, long InputQueuePosition, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(serviceInstanceId);
|
||||
this.WriteEvent(251, Account, TaskHub, PartitionId, CheckpointId, Reason, StoreStats, CommitLogPosition, InputQueuePosition, ExtensionVersion);
|
||||
}
|
||||
|
||||
[Event(252, Level = EventLevel.Informational, Version = 1)]
|
||||
public void FasterCheckpointPersisted(string Account, string TaskHub, int PartitionId, Guid CheckpointId, string Reason, long CommitLogPosition, long InputQueuePosition, long LatencyMs, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(serviceInstanceId);
|
||||
this.WriteEvent(252, Account, TaskHub, PartitionId, CheckpointId, Reason, CommitLogPosition, InputQueuePosition, LatencyMs, ExtensionVersion);
|
||||
}
|
||||
|
||||
[Event(253, Level = EventLevel.Verbose, Version = 1)]
|
||||
public void FasterLogPersisted(string Account, string TaskHub, int PartitionId, long CommitLogPosition, long NumberEvents, long SizeInBytes, long LatencyMs, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(serviceInstanceId);
|
||||
this.WriteEvent(253, Account, TaskHub, PartitionId, CommitLogPosition, NumberEvents, SizeInBytes, LatencyMs, ExtensionVersion);
|
||||
}
|
||||
|
||||
[Event(254, Level = EventLevel.Informational, Version = 1)]
|
||||
public void FasterCheckpointLoaded(string Account, string TaskHub, int PartitionId, long CommitLogPosition, long InputQueuePosition, string StoreStats, long LatencyMs, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(serviceInstanceId);
|
||||
this.WriteEvent(254, Account, TaskHub, PartitionId, CommitLogPosition, InputQueuePosition, StoreStats, LatencyMs, ExtensionVersion);
|
||||
}
|
||||
|
||||
[Event(255, Level = EventLevel.Informational, Version = 1)]
|
||||
public void FasterLogReplayed(string Account, string TaskHub, int PartitionId, long CommitLogPosition, long InputQueuePosition, long NumberEvents, long SizeInBytes, string StoreStats, long LatencyMs, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(serviceInstanceId);
|
||||
this.WriteEvent(255, Account, TaskHub, PartitionId, CommitLogPosition, InputQueuePosition, NumberEvents, SizeInBytes, StoreStats, LatencyMs, ExtensionVersion);
|
||||
}
|
||||
|
||||
[Event(256, Level = EventLevel.Error, Version = 1)]
|
||||
public void FasterStorageError(string Account, string TaskHub, int PartitionId, string Context, string Details, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(serviceInstanceId);
|
||||
this.WriteEvent(256, Account, TaskHub, PartitionId, Context, Details, ExtensionVersion);
|
||||
}
|
||||
|
||||
[Event(257, Level = EventLevel.Error, Version = 1)]
|
||||
public void FasterBlobStorageError(string Account, string TaskHub, int PartitionId, string Context, string BlobName, string Details, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(serviceInstanceId);
|
||||
this.WriteEvent(257, Account, TaskHub, PartitionId, Context, BlobName, Details, ExtensionVersion);
|
||||
}
|
||||
|
||||
[Event(258, Level = EventLevel.Warning, Version = 1)]
|
||||
public void FasterBlobStorageWarning(string Account, string TaskHub, int PartitionId, string Context, string BlobName, string Details, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(serviceInstanceId);
|
||||
this.WriteEvent(258, Account, TaskHub, PartitionId, Context, BlobName, Details, ExtensionVersion);
|
||||
}
|
||||
|
||||
[Event(259, Level = EventLevel.Verbose, Version = 1)]
|
||||
public void FasterProgress(string Account, string TaskHub, int PartitionId, string Details, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(serviceInstanceId);
|
||||
this.WriteEvent(259, Account, TaskHub, PartitionId, Details, ExtensionVersion);
|
||||
}
|
||||
|
||||
[Event(260, Level = EventLevel.Informational, Version = 1)]
|
||||
public void FasterLeaseAcquired(string Account, string TaskHub, int PartitionId, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(serviceInstanceId);
|
||||
this.WriteEvent(260, Account, TaskHub, PartitionId, ExtensionVersion);
|
||||
}
|
||||
|
||||
[Event(261, Level = EventLevel.Informational, Version = 1)]
|
||||
public void FasterLeaseReleased(string Account, string TaskHub, int PartitionId, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(serviceInstanceId);
|
||||
this.WriteEvent(261, Account, TaskHub, PartitionId, ExtensionVersion);
|
||||
}
|
||||
|
||||
[Event(262, Level = EventLevel.Warning, Version = 1)]
|
||||
public void FasterLeaseLost(string Account, string TaskHub, int PartitionId, string Details, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(serviceInstanceId);
|
||||
this.WriteEvent(262, Account, TaskHub, PartitionId, Details, ExtensionVersion);
|
||||
}
|
||||
|
||||
[Event(263, Level = EventLevel.Verbose, Version = 1)]
|
||||
public void FasterLeaseProgress(string Account, string TaskHub, int PartitionId, string Details, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(serviceInstanceId);
|
||||
this.WriteEvent(263, Account, TaskHub, PartitionId, Details, ExtensionVersion);
|
||||
}
|
||||
|
||||
[Event(264, Level = EventLevel.Verbose, Version = 1)]
|
||||
public void FasterStorageProgress(string Account, string TaskHub, int PartitionId, string Details, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(serviceInstanceId);
|
||||
this.WriteEvent(264, Account, TaskHub, PartitionId, Details, ExtensionVersion);
|
||||
}
|
||||
|
||||
[Event(265, Level = EventLevel.Warning, Version = 1)]
|
||||
public void FasterPerfWarning(string Account, string TaskHub, int PartitionId, string Details, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(serviceInstanceId);
|
||||
this.WriteEvent(265, Account, TaskHub, PartitionId, Details, ExtensionVersion);
|
||||
}
|
||||
|
||||
|
||||
// ----- EventHubs Transport
|
||||
|
||||
[Event(270, Level = EventLevel.Informational, Version = 1)]
|
||||
public void EventHubsInformation(string Account, string TaskHub, string EventHubsNamespace, string Details, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(serviceInstanceId);
|
||||
this.WriteEvent(270, Account, TaskHub, EventHubsNamespace, Details, ExtensionVersion);
|
||||
}
|
||||
|
||||
[Event(271, Level = EventLevel.Warning, Version = 1)]
|
||||
public void EventHubsWarning(string Account, string TaskHub, string EventHubsNamespace, string Details, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(serviceInstanceId);
|
||||
this.WriteEvent(271, Account, TaskHub, EventHubsNamespace, Details, ExtensionVersion);
|
||||
}
|
||||
|
||||
[Event(272, Level = EventLevel.Error, Version = 1)]
|
||||
public void EventHubsError(string Account, string TaskHub, string EventHubsNamespace, string Details, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(serviceInstanceId);
|
||||
this.WriteEvent(272, Account, TaskHub, EventHubsNamespace, Details, ExtensionVersion);
|
||||
}
|
||||
|
||||
[Event(273, Level = EventLevel.Verbose, Version = 1)]
|
||||
public void EventHubsDebug(string Account, string TaskHub, string EventHubsNamespace, string Details, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(serviceInstanceId);
|
||||
this.WriteEvent(273, Account, TaskHub, EventHubsNamespace, Details, ExtensionVersion);
|
||||
}
|
||||
|
||||
[Event(274, Level = EventLevel.Verbose, Version = 1)]
|
||||
public void EventHubsTrace(string Account, string TaskHub, string EventHubsNamespace, string Details, string ExtensionVersion)
|
||||
{
|
||||
SetCurrentThreadActivityId(serviceInstanceId);
|
||||
this.WriteEvent(274, Account, TaskHub, EventHubsNamespace, Details, ExtensionVersion);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
|
||||
static class EventTraceContext
|
||||
{
|
||||
[ThreadStatic]
|
||||
static (long commitLogPosition, string eventId) context;
|
||||
|
||||
static readonly TraceContextClear traceContextClear = new TraceContextClear();
|
||||
|
||||
public static IDisposable MakeContext(long commitLogPosition, string eventId)
|
||||
{
|
||||
EventTraceContext.context = (commitLogPosition, eventId);
|
||||
return traceContextClear;
|
||||
}
|
||||
|
||||
public static (long commitLogPosition, string eventId) Current => EventTraceContext.context;
|
||||
|
||||
class TraceContextClear : IDisposable
|
||||
{
|
||||
public void Dispose()
|
||||
{
|
||||
EventTraceContext.context = (0L, null);
|
||||
}
|
||||
}
|
||||
|
||||
public static void Clear()
|
||||
{
|
||||
EventTraceContext.context = (0L, null);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,385 @@
|
|||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using DurableTask.Core;
|
||||
using DurableTask.Core.History;
|
||||
using Dynamitey;
|
||||
using Dynamitey.DynamicObjects;
|
||||
using FASTER.core;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
class EventTraceHelper
|
||||
{
|
||||
readonly ILogger logger;
|
||||
readonly ILogger benchmarkLogger;
|
||||
readonly LogLevel logLevelLimit;
|
||||
readonly string account;
|
||||
readonly string taskHub;
|
||||
readonly int partitionId;
|
||||
readonly EtwSource etw;
|
||||
|
||||
public EventTraceHelper(ILoggerFactory loggerFactory, LogLevel logLevelLimit, Partition partition)
|
||||
{
|
||||
this.logger = loggerFactory.CreateLogger($"{NetheriteOrchestrationService.LoggerCategoryName}.Events");
|
||||
this.benchmarkLogger = loggerFactory.CreateLogger($"{NetheriteOrchestrationService.LoggerCategoryName}.Benchmark");
|
||||
this.logLevelLimit = logLevelLimit;
|
||||
this.account = partition.StorageAccountName;
|
||||
this.taskHub = partition.Settings.HubName;
|
||||
this.partitionId = (int)partition.PartitionId;
|
||||
this.etw = EtwSource.Log.IsEnabled() ? EtwSource.Log : null;
|
||||
}
|
||||
|
||||
public bool IsTracingAtMostDetailedLevel => this.logLevelLimit == LogLevel.Trace;
|
||||
|
||||
public void TraceEventProcessed(long commitLogPosition, PartitionEvent evt, double startedTimestamp, double finishedTimestamp, bool replaying)
|
||||
{
|
||||
if (this.logLevelLimit <= LogLevel.Information)
|
||||
{
|
||||
long nextCommitLogPosition = ((evt as PartitionUpdateEvent)?.NextCommitLogPosition) ?? 0L;
|
||||
double queueLatencyMs = evt.IssuedTimestamp - evt.ReceivedTimestamp;
|
||||
double fetchLatencyMs = startedTimestamp - evt.IssuedTimestamp;
|
||||
double latencyMs = finishedTimestamp - startedTimestamp;
|
||||
|
||||
if (this.logger.IsEnabled(LogLevel.Information))
|
||||
{
|
||||
var details = string.Format($"{(replaying ? "Replayed" : "Processed")} {(evt.NextInputQueuePosition > 0 ? "external" : "internal")} {(nextCommitLogPosition > 0 ? "update" : "read")} event");
|
||||
this.logger.LogInformation("Part{partition:D2}.{commitLogPosition:D10} {details} {event} eventId={eventId} pos=({nextCommitLogPosition},{nextInputQueuePosition}) latency=({queueLatencyMs:F0}, {fetchLatencyMs:F0}, {latencyMs:F0})", this.partitionId, commitLogPosition, details, evt, evt.EventIdString, nextCommitLogPosition, evt.NextInputQueuePosition, queueLatencyMs, fetchLatencyMs, latencyMs);
|
||||
}
|
||||
|
||||
this.etw?.PartitionEventProcessed(this.account, this.taskHub, this.partitionId, commitLogPosition, evt.EventIdString, evt.ToString(), nextCommitLogPosition, evt.NextInputQueuePosition, queueLatencyMs, fetchLatencyMs, latencyMs, replaying, TraceUtils.ExtensionVersion);
|
||||
}
|
||||
|
||||
// TODO remove this for production, it is here for benchmarking only
|
||||
//if (this.logLevelLimit <= LogLevel.Warning && !replaying)
|
||||
//{
|
||||
// // This is here for measuring response times
|
||||
// if (this.benchmarkLogger.IsEnabled(LogLevel.Warning))
|
||||
// {
|
||||
// string eventType;
|
||||
// switch (evt)
|
||||
// {
|
||||
// case CreationRequestReceived creationRequestEvent:
|
||||
// // Q: Is the receivedTimestamp the correct point to measure response time?
|
||||
// var startTimestamp = creationRequestEvent.ReceivedTimestamp;
|
||||
// var instanceId = creationRequestEvent.InstanceId;
|
||||
// eventType = "CreationRequestReceived";
|
||||
// this.benchmarkLogger.LogWarning("Part{partition:D2}.{commitLogPosition:D10} {eventType} for {instanceId} at {timestamp}", this.partitionId, commitLogPosition, eventType, instanceId, startTimestamp);
|
||||
// break;
|
||||
|
||||
// // Old way of measuring orchestration completion time that involves noise due to client polling times
|
||||
// //case StateRequestReceived readEvent:
|
||||
// // // Q: Is this how all our benchmarks end? It seems so.
|
||||
// // var endTimestamp = finishedTimestamp;
|
||||
// // var readTarget = readEvent.ReadTarget.InstanceId;
|
||||
// // eventType = "StateRequestReceived";
|
||||
// // this.logger.LogWarning("Part{partition:D2}.{commitLogPosition:D10} {eventType} for {instanceId} was processed at {timestamp}", this.partitionId, commitLogPosition, eventType, readTarget, endTimestamp);
|
||||
// // break;
|
||||
|
||||
// case BatchProcessed batchProcessedEvent:
|
||||
// var state = batchProcessedEvent.State;
|
||||
// if (state != null)
|
||||
// {
|
||||
// var instanceStatus = state.OrchestrationStatus;
|
||||
// if (instanceStatus == OrchestrationStatus.Canceled ||
|
||||
// instanceStatus == OrchestrationStatus.Completed ||
|
||||
// instanceStatus == OrchestrationStatus.Failed ||
|
||||
// instanceStatus == OrchestrationStatus.Terminated)
|
||||
// {
|
||||
// var targetInstanceId = batchProcessedEvent.InstanceId;
|
||||
// var finalTimestamp = finishedTimestamp;
|
||||
// var creationTime = batchProcessedEvent.State.CreatedTime.Ticks;
|
||||
// var completionTime = batchProcessedEvent.State.CompletedTime.Ticks;
|
||||
// var elapsedTime = new TimeSpan(completionTime - creationTime);
|
||||
// eventType = "BatchProcessed";
|
||||
// this.benchmarkLogger.LogWarning("Part{partition:D2}.{commitLogPosition:D10} {status} {eventType} for {instanceId} was processed at {timestamp}", this.partitionId, commitLogPosition, instanceStatus, eventType, targetInstanceId, finalTimestamp);
|
||||
// //this.logger.LogWarning("Part{partition:D2}.{commitLogPosition:D10} {instanceId} with creation time: {creationTime} and completionTime: {completionTime} was {status} at {timestamp}", this.partitionId, commitLogPosition, targetInstanceId, creationTime, completionTime, instanceStatus, finalTimestamp);
|
||||
// this.benchmarkLogger.LogWarning("Part{partition:D2}.{commitLogPosition:D10} {instanceId} took {elapsedMilliseconds} ms from its creation to completion with status: {status} at {timestamp}", this.partitionId, commitLogPosition, targetInstanceId, elapsedTime.TotalMilliseconds, instanceStatus, finalTimestamp);
|
||||
// }
|
||||
|
||||
// }
|
||||
// else
|
||||
// {
|
||||
// // This could be `null` if the instance is not executable (e.g. when the messages where for an existing completed executionID).
|
||||
// var targetInstanceId = batchProcessedEvent.InstanceId;
|
||||
// var eventIdString = batchProcessedEvent.EventIdString;
|
||||
// eventType = "BatchProcessed";
|
||||
// this.benchmarkLogger.LogWarning("Part{partition:D2}.{commitLogPosition:D10} State was NULL in {eventType}:{eventId} for {instanceId} that was processed at {timestamp}", this.partitionId, commitLogPosition, eventType, eventIdString, targetInstanceId, finishedTimestamp);
|
||||
// }
|
||||
// break;
|
||||
// }
|
||||
// }
|
||||
//}
|
||||
}
|
||||
|
||||
public void TraceTaskMessageReceived(TaskMessage message, string sessionPosition)
|
||||
{
|
||||
if (this.logLevelLimit <= LogLevel.Debug)
|
||||
{
|
||||
(long commitLogPosition, string eventId) = EventTraceContext.Current;
|
||||
|
||||
if (this.logger.IsEnabled(LogLevel.Debug))
|
||||
{
|
||||
string prefix = commitLogPosition > 0 ? $".{commitLogPosition:D10} " : "";
|
||||
this.logger.LogDebug("Part{partition:D2}{prefix} received TaskMessage eventType={eventType} taskEventId={taskEventId} instanceId={instanceId} executionId={executionId} messageId={messageId} sessionPosition={SessionPosition}",
|
||||
this.partitionId, prefix, message.Event.EventType.ToString(), TraceUtils.GetTaskEventId(message.Event), message.OrchestrationInstance.InstanceId, message.OrchestrationInstance.ExecutionId, eventId, sessionPosition);
|
||||
}
|
||||
|
||||
this.etw?.TaskMessageReceived(this.account, this.taskHub, this.partitionId, commitLogPosition, message.Event.EventType.ToString(), TraceUtils.GetTaskEventId(message.Event), message.OrchestrationInstance.InstanceId, message.OrchestrationInstance.ExecutionId ?? "", eventId, sessionPosition, TraceUtils.ExtensionVersion);
|
||||
}
|
||||
}
|
||||
|
||||
public void TraceTaskMessageSent(TaskMessage message, string sentEventId)
|
||||
{
|
||||
if (this.logLevelLimit <= LogLevel.Debug)
|
||||
{
|
||||
if (this.logger.IsEnabled(LogLevel.Debug))
|
||||
{
|
||||
(long commitLogPosition, string eventId) = EventTraceContext.Current;
|
||||
|
||||
string prefix = commitLogPosition > 0 ? $".{commitLogPosition:D10} " : "";
|
||||
this.logger.LogDebug("Part{partition:D2}{prefix} sent TaskMessage eventType={eventType} taskEventId={taskEventId} instanceId={instanceId} executionId={executionId} messageId={messageId}",
|
||||
this.partitionId, prefix, message.Event.EventType.ToString(), TraceUtils.GetTaskEventId(message.Event), message.OrchestrationInstance.InstanceId, message.OrchestrationInstance.ExecutionId, sentEventId);
|
||||
}
|
||||
|
||||
this.etw?.TaskMessageSent(this.account, this.taskHub, this.partitionId, message.Event.EventType.ToString(), TraceUtils.GetTaskEventId(message.Event), message.OrchestrationInstance.InstanceId, message.OrchestrationInstance.ExecutionId ?? "", sentEventId, TraceUtils.ExtensionVersion);
|
||||
}
|
||||
}
|
||||
|
||||
public void TraceTaskMessageDiscarded(TaskMessage message, string reason, string workItemId)
|
||||
{
|
||||
if (this.logLevelLimit <= LogLevel.Warning)
|
||||
{
|
||||
if (this.logger.IsEnabled(LogLevel.Warning))
|
||||
{
|
||||
(long commitLogPosition, string eventId) = EventTraceContext.Current;
|
||||
|
||||
string prefix = commitLogPosition > 0 ? $".{commitLogPosition:D10} " : "";
|
||||
this.logger.LogWarning("Part{partition:D2}{prefix} discarded TaskMessage reason={reason} eventType={eventType} taskEventId={taskEventId} instanceId={instanceId} executionId={executionId} workItemId={workItemId}",
|
||||
this.partitionId, prefix, reason, message.Event.EventType.ToString(), TraceUtils.GetTaskEventId(message.Event), message.OrchestrationInstance.InstanceId, message.OrchestrationInstance.ExecutionId, workItemId);
|
||||
}
|
||||
|
||||
this.etw?.TaskMessageDiscarded(this.account, this.taskHub, this.partitionId, reason, message.Event.EventType.ToString(), TraceUtils.GetTaskEventId(message.Event), message.OrchestrationInstance.InstanceId, message.OrchestrationInstance.ExecutionId ?? "", workItemId, TraceUtils.ExtensionVersion);
|
||||
}
|
||||
}
|
||||
|
||||
public void TraceOrchestrationWorkItemQueued(OrchestrationWorkItem workItem)
|
||||
{
|
||||
if (this.logLevelLimit <= LogLevel.Debug)
|
||||
{
|
||||
if (this.logger.IsEnabled(LogLevel.Debug))
|
||||
{
|
||||
(long commitLogPosition, string eventId) = EventTraceContext.Current;
|
||||
|
||||
string prefix = commitLogPosition > 0 ? $".{commitLogPosition:D10} " : "";
|
||||
this.logger.LogDebug("Part{partition:D2}{prefix} Queued {executionType} OrchestrationWorkItem {workItemId} instanceId={instanceId} ",
|
||||
this.partitionId, prefix, workItem.Type, workItem.MessageBatch.WorkItemId, workItem.InstanceId);
|
||||
}
|
||||
|
||||
this.etw?.OrchestrationWorkItemQueued(this.account, this.taskHub, this.partitionId, workItem.Type.ToString(), workItem.InstanceId, workItem.MessageBatch.WorkItemId, TraceUtils.ExtensionVersion);
|
||||
}
|
||||
}
|
||||
|
||||
public void TraceOrchestrationWorkItemDiscarded(BatchProcessed evt)
|
||||
{
|
||||
if (this.logLevelLimit <= LogLevel.Warning)
|
||||
{
|
||||
if (this.logger.IsEnabled(LogLevel.Warning))
|
||||
{
|
||||
(long commitLogPosition, string eventId) = EventTraceContext.Current;
|
||||
|
||||
string prefix = commitLogPosition > 0 ? $".{commitLogPosition:D10} " : "";
|
||||
this.logger.LogWarning("Part{partition:D2}{prefix} Discarded OrchestrationWorkItem workItemId={workItemId} instanceId={instanceId}",
|
||||
this.partitionId, prefix, evt.WorkItemId, evt.InstanceId);
|
||||
}
|
||||
|
||||
this.etw?.OrchestrationWorkItemDiscarded(this.account, this.taskHub, this.partitionId, evt.InstanceId, evt.WorkItemId, TraceUtils.ExtensionVersion);
|
||||
}
|
||||
}
|
||||
|
||||
public void TraceInstanceUpdate(string workItemId, string instanceId, string executionId, int totalEventCount, List<HistoryEvent> newEvents, int episode)
|
||||
{
|
||||
if (this.logLevelLimit <= LogLevel.Debug)
|
||||
{
|
||||
string eventNames = string.Empty;
|
||||
string eventType = string.Empty;
|
||||
int numNewEvents = 0;
|
||||
|
||||
if (newEvents != null)
|
||||
{
|
||||
HistoryEvent orchestratorEvent = null;
|
||||
var eventNamesBuilder = new StringBuilder();
|
||||
numNewEvents = newEvents.Count;
|
||||
|
||||
if (numNewEvents > 20)
|
||||
{
|
||||
eventNamesBuilder.Append("...,");
|
||||
}
|
||||
|
||||
for (int i = 0; i < numNewEvents; i++)
|
||||
{
|
||||
var historyEvent = newEvents[i];
|
||||
switch (historyEvent.EventType)
|
||||
{
|
||||
case EventType.ExecutionStarted:
|
||||
case EventType.ExecutionCompleted:
|
||||
case EventType.ExecutionTerminated:
|
||||
case EventType.ContinueAsNew:
|
||||
orchestratorEvent = historyEvent;
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
if (i >= newEvents.Count - 20)
|
||||
{
|
||||
eventNamesBuilder.Append(historyEvent.EventType.ToString());
|
||||
eventNamesBuilder.Append(",");
|
||||
}
|
||||
}
|
||||
eventNames = eventNamesBuilder.ToString(0, eventNamesBuilder.Length - 1); // remove trailing comma
|
||||
if (orchestratorEvent != null)
|
||||
{
|
||||
eventType = orchestratorEvent.EventType.ToString();
|
||||
}
|
||||
}
|
||||
|
||||
if (this.logger.IsEnabled(LogLevel.Debug))
|
||||
{
|
||||
(long commitLogPosition, string eventId) = EventTraceContext.Current;
|
||||
|
||||
string prefix = commitLogPosition > 0 ? $".{commitLogPosition:D10} " : "";
|
||||
this.logger.LogDebug("Part{partition:D2}{prefix} Updated instance instanceId={instanceId} executionId={executionId} workItemId={workItemId} numNewEvents={numNewEvents} totalEventCount={totalEventCount} eventNames={eventNames} eventType={eventType} episode={episode}",
|
||||
this.partitionId, prefix, instanceId, executionId, workItemId, numNewEvents, totalEventCount, eventNames, eventType, episode);
|
||||
}
|
||||
|
||||
this.etw?.InstanceUpdated(this.account, this.taskHub, this.partitionId, instanceId, executionId, workItemId, numNewEvents, totalEventCount, eventNames, eventType, episode, TraceUtils.ExtensionVersion);
|
||||
}
|
||||
}
|
||||
|
||||
public void TraceFetchedInstanceStatus(PartitionReadEvent evt, string instanceId, string executionId, string status, double latencyMs)
|
||||
{
|
||||
if (this.logLevelLimit <= LogLevel.Debug)
|
||||
{
|
||||
if (this.logger.IsEnabled(LogLevel.Debug))
|
||||
{
|
||||
(long commitLogPosition, string eventId) = EventTraceContext.Current;
|
||||
|
||||
string prefix = commitLogPosition > 0 ? $".{commitLogPosition:D10} " : "";
|
||||
this.logger.LogDebug("Part{partition:D2}{prefix} Fetched instance status instanceId={instanceId} executionId={executionId} status={status} eventId={eventId} latencyMs={latencyMs:F0}",
|
||||
this.partitionId, prefix, instanceId, executionId, status, evt.EventIdString, latencyMs);
|
||||
}
|
||||
|
||||
this.etw?.InstanceStatusFetched(this.account, this.taskHub, this.partitionId, instanceId, executionId, status, evt.EventIdString, latencyMs, TraceUtils.ExtensionVersion);
|
||||
}
|
||||
}
|
||||
|
||||
public void TraceFetchedInstanceHistory(PartitionReadEvent evt, string instanceId, string executionId, int eventCount, int episode, double latencyMs)
|
||||
{
|
||||
if (this.logLevelLimit <= LogLevel.Debug)
|
||||
{
|
||||
if (this.logger.IsEnabled(LogLevel.Debug))
|
||||
{
|
||||
(long commitLogPosition, string eventId) = EventTraceContext.Current;
|
||||
|
||||
string prefix = commitLogPosition > 0 ? $".{commitLogPosition:D10} " : "";
|
||||
this.logger.LogDebug("Part{partition:D2}{prefix} Fetched instance history instanceId={instanceId} executionId={executionId} eventCount={eventCount} episode={episode} eventId={eventId} latencyMs={latencyMs:F0}",
|
||||
this.partitionId, prefix, instanceId, executionId, eventCount, episode, evt.EventIdString, latencyMs);
|
||||
}
|
||||
|
||||
this.etw?.InstanceHistoryFetched(this.account, this.taskHub, this.partitionId, instanceId, executionId ?? string.Empty, eventCount, episode, evt.EventIdString, latencyMs, TraceUtils.ExtensionVersion);
|
||||
}
|
||||
}
|
||||
|
||||
public void TracePartitionOffloadDecision(int reportedLocalLoad, int pending, int backlog, int remotes, string reportedRemotes)
|
||||
{
|
||||
if (this.logLevelLimit <= LogLevel.Warning)
|
||||
{
|
||||
if (this.logLevelLimit <= LogLevel.Warning)
|
||||
{
|
||||
(long commitLogPosition, string eventId) = EventTraceContext.Current;
|
||||
|
||||
string prefix = commitLogPosition > 0 ? $".{commitLogPosition:D10} " : "";
|
||||
this.logger.LogWarning("Part{partition:D2}{prefix} Offload decision reportedLocalLoad={reportedLocalLoad} pending={pending} backlog={backlog} remotes={remotes} reportedRemotes={reportedRemotes}",
|
||||
this.partitionId, prefix, reportedLocalLoad, pending, backlog, remotes, reportedRemotes);
|
||||
|
||||
this.etw?.PartitionOffloadDecision(this.account, this.taskHub, this.partitionId, commitLogPosition, eventId, reportedLocalLoad, pending, backlog, remotes, reportedRemotes, TraceUtils.ExtensionVersion);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void TraceEventProcessingStarted(long commitLogPosition, PartitionEvent evt, bool replaying)
|
||||
{
|
||||
if (this.logLevelLimit <= LogLevel.Trace)
|
||||
{
|
||||
long nextCommitLogPosition = ((evt as PartitionUpdateEvent)?.NextCommitLogPosition) ?? 0L;
|
||||
var details = string.Format($"{(replaying ? "Replaying" : "Processing")} {(evt.NextInputQueuePosition > 0 ? "external" : "internal")} {(nextCommitLogPosition > 0 ? "update" : "read")} event {evt} id={evt.EventIdString}");
|
||||
if (this.logger.IsEnabled(LogLevel.Trace))
|
||||
{
|
||||
this.logger.LogTrace("Part{partition:D2}.{commitLogPosition:D10} {details}", this.partitionId, commitLogPosition, details);
|
||||
}
|
||||
this.etw?.PartitionEventDetail(this.account, this.taskHub, this.partitionId, commitLogPosition, evt.EventIdString ?? "", details, TraceUtils.ExtensionVersion);
|
||||
}
|
||||
}
|
||||
|
||||
public void TraceEventProcessingDetail(string details)
|
||||
{
|
||||
if (this.logLevelLimit <= LogLevel.Trace)
|
||||
{
|
||||
(long commitLogPosition, string eventId) = EventTraceContext.Current;
|
||||
if (this.logger.IsEnabled(LogLevel.Trace))
|
||||
{
|
||||
string prefix = commitLogPosition > 0 ? $".{commitLogPosition:D10} " : "";
|
||||
this.logger.LogTrace("Part{partition:D2}{prefix} {details}", this.partitionId, prefix, details);
|
||||
}
|
||||
this.etw?.PartitionEventDetail(this.account, this.taskHub, this.partitionId, commitLogPosition, eventId ?? "", details, TraceUtils.ExtensionVersion);
|
||||
}
|
||||
}
|
||||
|
||||
public void TraceEventSentDetail(PartitionUpdateEvent evt)
|
||||
{
|
||||
if (this.logLevelLimit <= LogLevel.Debug)
|
||||
{
|
||||
(long commitLogPosition, string eventId) = EventTraceContext.Current;
|
||||
double sentTimestamp = evt.SentTimestamp;
|
||||
double noSpeculationDelay = sentTimestamp - ((PartitionUpdateEvent)evt).ReadyToSendTimestamp;
|
||||
if (this.logger.IsEnabled(LogLevel.Debug))
|
||||
{
|
||||
string prefix = commitLogPosition > 0 ? $".{commitLogPosition:D10} " : "";
|
||||
this.logger.LogDebug("Part{partition:D2}{prefix} Event sent with delay: {noSpeculationDelay} ms", this.partitionId, prefix, noSpeculationDelay);
|
||||
}
|
||||
this.etw?.PartitionEventDetail(this.account, this.taskHub, this.partitionId, commitLogPosition, eventId ?? "", $"Event sent with delay: {noSpeculationDelay} ms", TraceUtils.ExtensionVersion);
|
||||
}
|
||||
}
|
||||
|
||||
public void TraceWarning(string details)
|
||||
{
|
||||
if (this.logLevelLimit <= LogLevel.Warning)
|
||||
{
|
||||
(long commitLogPosition, string eventId) = EventTraceContext.Current;
|
||||
if (this.logger.IsEnabled(LogLevel.Warning))
|
||||
{
|
||||
string prefix = commitLogPosition > 0 ? $".{commitLogPosition:D10} " : "";
|
||||
this.logger.LogWarning("Part{partition:D2}{prefix} {details}", this.partitionId, prefix, details);
|
||||
}
|
||||
|
||||
// for now no ETW, TODO
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,104 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Text;
|
||||
using DurableTask.Core;
|
||||
|
||||
/// <summary>
|
||||
/// Schema for instance queries
|
||||
/// </summary>
|
||||
[DataContract]
|
||||
public class InstanceQuery
|
||||
{
|
||||
/// <summary>
|
||||
/// The subset of runtime statuses to return, or null if all
|
||||
/// </summary>
|
||||
[DataMember]
|
||||
public OrchestrationStatus[] RuntimeStatus { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The lowest creation time to return, or null if no lower bound
|
||||
/// </summary>
|
||||
[DataMember]
|
||||
public DateTime? CreatedTimeFrom { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The latest creation time to return, or null if no upper bound
|
||||
/// </summary>
|
||||
[DataMember]
|
||||
public DateTime? CreatedTimeTo { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// A prefix of the instance ids to return, or null if no specific prefix
|
||||
/// </summary>
|
||||
[DataMember]
|
||||
public string InstanceIdPrefix { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether to fetch the input along with the orchestration state.
|
||||
/// </summary>
|
||||
[DataMember]
|
||||
public bool FetchInput { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether to prefetch the history for all returned instances.
|
||||
/// </summary>
|
||||
[DataMember]
|
||||
internal bool PrefetchHistory { get; set; }
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Construct an instance query with the given parameters.
|
||||
/// </summary>
|
||||
/// <param name="runtimeStatus">The subset of runtime statuses to return, or null if all</param>
|
||||
/// <param name="createdTimeFrom">The lowest creation time to return, or null if no lower bound.</param>
|
||||
/// <param name="createdTimeTo">The latest creation time to return, or null if no upper bound.</param>
|
||||
/// <param name="instanceIdPrefix">A prefix of the instance ids to return, or null if no specific prefix.</param>
|
||||
/// <param name="fetchInput">Whether to fetch the input along with the orchestration state.</param>
|
||||
public InstanceQuery(
|
||||
OrchestrationStatus[] runtimeStatus = null,
|
||||
DateTime? createdTimeFrom = null,
|
||||
DateTime? createdTimeTo = null,
|
||||
string instanceIdPrefix = null,
|
||||
bool fetchInput = true)
|
||||
{
|
||||
this.RuntimeStatus = runtimeStatus;
|
||||
this.CreatedTimeFrom = createdTimeFrom;
|
||||
this.CreatedTimeTo = createdTimeTo;
|
||||
this.InstanceIdPrefix = instanceIdPrefix;
|
||||
this.FetchInput = fetchInput;
|
||||
}
|
||||
|
||||
internal bool HasRuntimeStatus => this.RuntimeStatus != null && this.RuntimeStatus.Length > 0;
|
||||
|
||||
internal bool IsSet => this.HasRuntimeStatus || !string.IsNullOrWhiteSpace(this.InstanceIdPrefix)
|
||||
|| !(this.CreatedTimeFrom is null) || !(this.CreatedTimeTo is null);
|
||||
|
||||
internal bool Matches(OrchestrationState targetState)
|
||||
{
|
||||
if (targetState == null)
|
||||
throw new ArgumentNullException(nameof(targetState));
|
||||
|
||||
return (!this.HasRuntimeStatus || this.RuntimeStatus.Contains(targetState.OrchestrationStatus))
|
||||
&& (string.IsNullOrWhiteSpace(this.InstanceIdPrefix) || targetState.OrchestrationInstance.InstanceId.StartsWith(this.InstanceIdPrefix))
|
||||
&& (!this.CreatedTimeFrom.HasValue || targetState.CreatedTime >= this.CreatedTimeFrom.Value)
|
||||
&& (!this.CreatedTimeTo.HasValue || targetState.CreatedTime <= this.CreatedTimeTo.Value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Text;
|
||||
using DurableTask.Core;
|
||||
|
||||
/// <summary>
|
||||
/// The result returned by an instance query
|
||||
/// </summary>
|
||||
[DataContract]
|
||||
public class InstanceQueryResult
|
||||
{
|
||||
/// <summary>
|
||||
/// The instances returned by the query.
|
||||
/// </summary>
|
||||
public IEnumerable<OrchestrationState> Instances { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// A continuation token to resume the query, or null if the results are complete.
|
||||
/// </summary>
|
||||
public string ContinuationToken { get; set; }
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,674 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using DurableTask.Core;
|
||||
using DurableTask.Core.Common;
|
||||
using DurableTask.Core.History;
|
||||
using DurableTask.Netherite.Faster;
|
||||
using DurableTask.Netherite.Scaling;
|
||||
using Microsoft.Azure.Storage;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Newtonsoft.Json;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
/// <summary>
|
||||
/// Local partition of the distributed orchestration service.
|
||||
/// </summary>
|
||||
public class NetheriteOrchestrationService :
|
||||
IOrchestrationService,
|
||||
IOrchestrationServiceClient,
|
||||
TransportAbstraction.IHost,
|
||||
IStorageProvider,
|
||||
IDisposable
|
||||
{
|
||||
readonly ITaskHub taskHub;
|
||||
readonly TransportConnectionString.StorageChoices configuredStorage;
|
||||
readonly TransportConnectionString.TransportChoices configuredTransport;
|
||||
|
||||
/// <summary>
|
||||
/// The logger category prefix used for all ILoggers in this backend.
|
||||
/// </summary>
|
||||
public const string LoggerCategoryName = "DurableTask.Netherite";
|
||||
|
||||
CancellationTokenSource serviceShutdownSource;
|
||||
|
||||
//internal Dictionary<uint, Partition> Partitions { get; private set; }
|
||||
internal Client Client { get; private set; }
|
||||
|
||||
internal ILoadMonitorService LoadMonitorService { get; private set; }
|
||||
|
||||
internal NetheriteOrchestrationServiceSettings Settings { get; private set; }
|
||||
internal uint NumberPartitions { get; private set; }
|
||||
uint TransportAbstraction.IHost.NumberPartitions { set => this.NumberPartitions = value; }
|
||||
internal string StorageAccountName { get; private set; }
|
||||
|
||||
internal WorkItemQueue<ActivityWorkItem> ActivityWorkItemQueue { get; private set; }
|
||||
internal WorkItemQueue<OrchestrationWorkItem> OrchestrationWorkItemQueue { get; private set; }
|
||||
internal LoadPublisher LoadPublisher { get; private set; }
|
||||
|
||||
internal Guid ServiceInstanceId { get; } = Guid.NewGuid();
|
||||
internal ILogger Logger { get; }
|
||||
internal ILoggerFactory LoggerFactory { get; }
|
||||
|
||||
/// <inheritdoc/>
|
||||
public override string ToString()
|
||||
{
|
||||
return $"NetheriteOrchestrationService on {this.configuredTransport}Transport and {this.configuredStorage}Storage";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new instance of the OrchestrationService with default settings
|
||||
/// </summary>
|
||||
public NetheriteOrchestrationService(NetheriteOrchestrationServiceSettings settings, ILoggerFactory loggerFactory)
|
||||
{
|
||||
this.Settings = settings;
|
||||
TransportConnectionString.Parse(this.Settings.EventHubsConnectionString, out this.configuredStorage, out this.configuredTransport, out _);
|
||||
this.Logger = loggerFactory.CreateLogger(LoggerCategoryName);
|
||||
this.LoggerFactory = loggerFactory;
|
||||
this.StorageAccountName = this.configuredTransport == TransportConnectionString.TransportChoices.Memory
|
||||
? this.Settings.StorageConnectionString
|
||||
: CloudStorageAccount.Parse(this.Settings.StorageConnectionString).Credentials.AccountName;
|
||||
|
||||
EtwSource.Log.OrchestrationServiceCreated(this.ServiceInstanceId, this.StorageAccountName, this.Settings.HubName, this.Settings.WorkerId, TraceUtils.ExtensionVersion);
|
||||
this.Logger.LogInformation("NetheriteOrchestrationService created, workerId={workerId}, transport={transport}, storage={storage}", this.Settings.WorkerId, this.configuredTransport, this.configuredStorage);
|
||||
|
||||
switch (this.configuredTransport)
|
||||
{
|
||||
case TransportConnectionString.TransportChoices.Memory:
|
||||
this.taskHub = new Emulated.MemoryTransport(this, settings, this.Logger);
|
||||
break;
|
||||
|
||||
case TransportConnectionString.TransportChoices.EventHubs:
|
||||
this.taskHub = new EventHubs.EventHubsTransport(this, settings, loggerFactory);
|
||||
break;
|
||||
|
||||
default:
|
||||
throw new NotImplementedException("no such transport choice");
|
||||
}
|
||||
|
||||
if (this.configuredTransport != TransportConnectionString.TransportChoices.Memory)
|
||||
this.LoadMonitorService = new AzureLoadMonitorTable(settings.StorageConnectionString, settings.LoadInformationAzureTableName, settings.HubName);
|
||||
|
||||
this.Logger.LogInformation(
|
||||
"trace generation limits: general={general} , transport={transport}, storage={storage}, events={events}; etwEnabled={etwEnabled}; core.IsTraceEnabled={core}",
|
||||
settings.LogLevelLimit,
|
||||
settings.TransportLogLevelLimit,
|
||||
settings.StorageLogLevelLimit,
|
||||
settings.EventLogLevelLimit,
|
||||
EtwSource.Log.IsEnabled(),
|
||||
DurableTask.Core.Tracing.DefaultEventSource.Log.IsTraceEnabled);
|
||||
}
|
||||
|
||||
/******************************/
|
||||
// storage provider
|
||||
/******************************/
|
||||
|
||||
IPartitionState IStorageProvider.CreatePartitionState()
|
||||
{
|
||||
switch (this.configuredStorage)
|
||||
{
|
||||
case TransportConnectionString.StorageChoices.Memory:
|
||||
return new MemoryStorage(this.Logger);
|
||||
|
||||
case TransportConnectionString.StorageChoices.Faster:
|
||||
return new Faster.FasterStorage(this.Settings.StorageConnectionString, this.Settings.PremiumStorageConnectionString, this.Settings.HubName, this.LoggerFactory);
|
||||
|
||||
default:
|
||||
throw new NotImplementedException("no such storage choice");
|
||||
}
|
||||
}
|
||||
|
||||
async Task IStorageProvider.DeleteAllPartitionStatesAsync()
|
||||
{
|
||||
if (!(this.LoadMonitorService is null))
|
||||
await this.LoadMonitorService.DeleteIfExistsAsync(CancellationToken.None).ConfigureAwait(false);
|
||||
|
||||
switch (this.configuredStorage)
|
||||
{
|
||||
case TransportConnectionString.StorageChoices.Memory:
|
||||
await Task.Delay(10).ConfigureAwait(false);
|
||||
break;
|
||||
|
||||
case TransportConnectionString.StorageChoices.Faster:
|
||||
await Faster.FasterStorage.DeleteTaskhubStorageAsync(this.Settings.StorageConnectionString, this.Settings.HubName).ConfigureAwait(false);
|
||||
break;
|
||||
|
||||
default:
|
||||
throw new NotImplementedException("no such storage choice");
|
||||
}
|
||||
}
|
||||
|
||||
/******************************/
|
||||
// management methods
|
||||
/******************************/
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task CreateAsync() => await ((IOrchestrationService)this).CreateAsync(true).ConfigureAwait(false);
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task CreateAsync(bool recreateInstanceStore)
|
||||
{
|
||||
if (await this.taskHub.ExistsAsync().ConfigureAwait(false))
|
||||
{
|
||||
if (recreateInstanceStore)
|
||||
{
|
||||
await this.taskHub.DeleteAsync().ConfigureAwait(false);
|
||||
await this.taskHub.CreateAsync().ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
await this.taskHub.CreateAsync().ConfigureAwait(false);
|
||||
}
|
||||
|
||||
if (!(this.LoadMonitorService is null))
|
||||
await this.LoadMonitorService.CreateIfNotExistsAsync(CancellationToken.None).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task CreateIfNotExistsAsync() => await ((IOrchestrationService)this).CreateAsync(false).ConfigureAwait(false);
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task DeleteAsync()
|
||||
{
|
||||
await this.taskHub.DeleteAsync().ConfigureAwait(false);
|
||||
|
||||
if (!(this.LoadMonitorService is null))
|
||||
await this.LoadMonitorService.DeleteIfExistsAsync(CancellationToken.None).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task DeleteAsync(bool deleteInstanceStore) => await this.DeleteAsync().ConfigureAwait(false);
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task StartAsync()
|
||||
{
|
||||
if (this.serviceShutdownSource != null)
|
||||
{
|
||||
// we left the service running. No need to start it again.
|
||||
return;
|
||||
}
|
||||
|
||||
this.Logger.LogInformation("NetheriteOrchestrationService is starting on workerId={workerId}", this.Settings.WorkerId);
|
||||
|
||||
this.serviceShutdownSource = new CancellationTokenSource();
|
||||
|
||||
this.ActivityWorkItemQueue = new WorkItemQueue<ActivityWorkItem>();
|
||||
this.OrchestrationWorkItemQueue = new WorkItemQueue<OrchestrationWorkItem>();
|
||||
|
||||
LeaseTimer.Instance.DelayWarning = (int delay) =>
|
||||
this.Logger.LogWarning("NetheriteOrchestrationService lease timer on workerId={workerId} is running {delay}s behind schedule", this.Settings.WorkerId, delay);
|
||||
|
||||
if (!(this.LoadMonitorService is null))
|
||||
this.LoadPublisher = new LoadPublisher(this.LoadMonitorService, this.Logger);
|
||||
|
||||
await this.taskHub.StartAsync().ConfigureAwait(false);
|
||||
|
||||
System.Diagnostics.Debug.Assert(this.Client != null, "Backend should have added client");
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task StopAsync(bool isForced)
|
||||
{
|
||||
this.Logger.LogInformation("NetheriteOrchestrationService stopping, workerId={workerId}", this.Settings.WorkerId);
|
||||
|
||||
if (!this.Settings.KeepServiceRunning && this.serviceShutdownSource != null)
|
||||
{
|
||||
this.serviceShutdownSource.Cancel();
|
||||
this.serviceShutdownSource.Dispose();
|
||||
this.serviceShutdownSource = null;
|
||||
|
||||
await this.taskHub.StopAsync(isForced).ConfigureAwait(false);
|
||||
|
||||
this.ActivityWorkItemQueue.Dispose();
|
||||
this.OrchestrationWorkItemQueue.Dispose();
|
||||
|
||||
this.Logger.LogInformation("NetheriteOrchestrationService stopped, workerId={workerId}", this.Settings.WorkerId);
|
||||
EtwSource.Log.OrchestrationServiceStopped(this.ServiceInstanceId, this.StorageAccountName, this.Settings.HubName, this.Settings.WorkerId, TraceUtils.ExtensionVersion);
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task StopAsync() => ((IOrchestrationService)this).StopAsync(false);
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void Dispose() => this.taskHub.StopAsync(true);
|
||||
|
||||
/// <summary>
|
||||
/// Computes the partition for the given instance.
|
||||
/// </summary>
|
||||
/// <param name="instanceId">The instance id.</param>
|
||||
/// <returns>The partition id.</returns>
|
||||
public uint GetPartitionId(string instanceId)
|
||||
{
|
||||
// if the instance id ends with !nn, where nn is a two-digit number, it indicates explicit partition placement
|
||||
if (instanceId.Length >= 3
|
||||
&& instanceId[instanceId.Length - 3] == '!'
|
||||
&& uint.TryParse(instanceId.Substring(instanceId.Length - 2), out uint nn))
|
||||
{
|
||||
var partitionId = nn % this.NumberPartitions;
|
||||
//this.Logger.LogTrace($"Instance: {instanceId} was explicitly placed on partition: {partitionId}");
|
||||
return partitionId;
|
||||
}
|
||||
else
|
||||
{
|
||||
return Fnv1aHashHelper.ComputeHash(instanceId) % this.NumberPartitions;
|
||||
}
|
||||
}
|
||||
|
||||
uint GetNumberPartitions() => this.NumberPartitions;
|
||||
|
||||
/******************************/
|
||||
// host methods
|
||||
/******************************/
|
||||
|
||||
TransportAbstraction.IClient TransportAbstraction.IHost.AddClient(Guid clientId, Guid taskHubGuid, TransportAbstraction.ISender batchSender)
|
||||
{
|
||||
System.Diagnostics.Debug.Assert(this.Client == null, "Backend should create only 1 client");
|
||||
|
||||
this.Client = new Client(this, clientId, taskHubGuid, batchSender, this.serviceShutdownSource.Token);
|
||||
return this.Client;
|
||||
}
|
||||
|
||||
TransportAbstraction.IPartition TransportAbstraction.IHost.AddPartition(uint partitionId, TransportAbstraction.ISender batchSender)
|
||||
{
|
||||
var partition = new Partition(this, partitionId, this.GetPartitionId, this.GetNumberPartitions, batchSender, this.Settings, this.StorageAccountName,
|
||||
this.ActivityWorkItemQueue, this.OrchestrationWorkItemQueue, this.LoadPublisher);
|
||||
|
||||
return partition;
|
||||
}
|
||||
|
||||
IStorageProvider TransportAbstraction.IHost.StorageProvider => this;
|
||||
|
||||
IPartitionErrorHandler TransportAbstraction.IHost.CreateErrorHandler(uint partitionId)
|
||||
{
|
||||
return new PartitionErrorHandler((int) partitionId, this.Logger, this.Settings.LogLevelLimit, this.StorageAccountName, this.Settings.HubName);
|
||||
}
|
||||
|
||||
/******************************/
|
||||
// client methods
|
||||
/******************************/
|
||||
|
||||
/// <inheritdoc />
|
||||
Task IOrchestrationServiceClient.CreateTaskOrchestrationAsync(TaskMessage creationMessage)
|
||||
=> this.Client.CreateTaskOrchestrationAsync(
|
||||
this.GetPartitionId(creationMessage.OrchestrationInstance.InstanceId),
|
||||
creationMessage,
|
||||
null);
|
||||
|
||||
/// <inheritdoc />
|
||||
Task IOrchestrationServiceClient.CreateTaskOrchestrationAsync(TaskMessage creationMessage, OrchestrationStatus[] dedupeStatuses)
|
||||
=> this.Client.CreateTaskOrchestrationAsync(
|
||||
this.GetPartitionId(creationMessage.OrchestrationInstance.InstanceId),
|
||||
creationMessage,
|
||||
dedupeStatuses);
|
||||
|
||||
/// <inheritdoc />
|
||||
Task IOrchestrationServiceClient.SendTaskOrchestrationMessageAsync(TaskMessage message)
|
||||
=> this.Client.SendTaskOrchestrationMessageBatchAsync(
|
||||
this.GetPartitionId(message.OrchestrationInstance.InstanceId),
|
||||
new[] { message });
|
||||
|
||||
/// <inheritdoc />
|
||||
Task IOrchestrationServiceClient.SendTaskOrchestrationMessageBatchAsync(params TaskMessage[] messages)
|
||||
=> messages.Length == 0
|
||||
? Task.CompletedTask
|
||||
: Task.WhenAll(messages
|
||||
.GroupBy(tm => this.GetPartitionId(tm.OrchestrationInstance.InstanceId))
|
||||
.Select(group => this.Client.SendTaskOrchestrationMessageBatchAsync(group.Key, group)));
|
||||
|
||||
/// <inheritdoc />
|
||||
Task<OrchestrationState> IOrchestrationServiceClient.WaitForOrchestrationAsync(
|
||||
string instanceId,
|
||||
string executionId,
|
||||
TimeSpan timeout,
|
||||
CancellationToken cancellationToken)
|
||||
=> this.Client.WaitForOrchestrationAsync(
|
||||
this.GetPartitionId(instanceId),
|
||||
instanceId,
|
||||
executionId,
|
||||
timeout,
|
||||
cancellationToken);
|
||||
|
||||
/// <inheritdoc />
|
||||
async Task<OrchestrationState> IOrchestrationServiceClient.GetOrchestrationStateAsync(
|
||||
string instanceId,
|
||||
string executionId)
|
||||
{
|
||||
var state = await this.Client.GetOrchestrationStateAsync(this.GetPartitionId(instanceId), instanceId, true).ConfigureAwait(false);
|
||||
return state != null && (executionId == null || executionId == state.OrchestrationInstance.ExecutionId)
|
||||
? state
|
||||
: null;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
async Task<IList<OrchestrationState>> IOrchestrationServiceClient.GetOrchestrationStateAsync(
|
||||
string instanceId,
|
||||
bool allExecutions)
|
||||
{
|
||||
// note: allExecutions is always ignored because storage contains never more than one execution.
|
||||
var state = await this.Client.GetOrchestrationStateAsync(this.GetPartitionId(instanceId), instanceId, true).ConfigureAwait(false);
|
||||
return state != null
|
||||
? (new[] { state })
|
||||
: (new OrchestrationState[0]);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
Task IOrchestrationServiceClient.ForceTerminateTaskOrchestrationAsync(
|
||||
string instanceId,
|
||||
string message)
|
||||
=> this.Client.ForceTerminateTaskOrchestrationAsync(this.GetPartitionId(instanceId), instanceId, message);
|
||||
|
||||
/// <inheritdoc />
|
||||
async Task<string> IOrchestrationServiceClient.GetOrchestrationHistoryAsync(
|
||||
string instanceId,
|
||||
string executionId)
|
||||
{
|
||||
(string actualExecutionId, IList<HistoryEvent> history) =
|
||||
await this.Client.GetOrchestrationHistoryAsync(this.GetPartitionId(instanceId), instanceId).ConfigureAwait(false);
|
||||
|
||||
if (history != null && (executionId == null || executionId == actualExecutionId))
|
||||
{
|
||||
return JsonConvert.SerializeObject(history);
|
||||
}
|
||||
else
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
Task IOrchestrationServiceClient.PurgeOrchestrationHistoryAsync(
|
||||
DateTime thresholdDateTimeUtc,
|
||||
OrchestrationStateTimeRangeFilterType
|
||||
timeRangeFilterType)
|
||||
{
|
||||
if (timeRangeFilterType != OrchestrationStateTimeRangeFilterType.OrchestrationCreatedTimeFilter)
|
||||
{
|
||||
throw new NotSupportedException("Purging is supported only for Orchestration created time filter.");
|
||||
}
|
||||
|
||||
return this.Client.PurgeInstanceHistoryAsync(thresholdDateTimeUtc, null, null);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the current state of an instance.
|
||||
/// </summary>
|
||||
/// <param name="instanceId">Instance ID of the orchestration.</param>
|
||||
/// <param name="fetchInput">If set, fetch and return the input for the orchestration instance.</param>
|
||||
/// <param name="fetchOutput">If set, fetch and return the output for the orchestration instance.</param>
|
||||
/// <returns>The state of the instance, or null if not found.</returns>
|
||||
public Task<OrchestrationState> GetOrchestrationStateAsync(string instanceId, bool fetchInput = true, bool fetchOutput = true)
|
||||
{
|
||||
return this.Client.GetOrchestrationStateAsync(this.GetPartitionId(instanceId), instanceId, fetchInput, fetchOutput);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the state of all orchestration instances.
|
||||
/// </summary>
|
||||
/// <returns>List of <see cref="OrchestrationState"/></returns>
|
||||
public Task<IList<OrchestrationState>> GetAllOrchestrationStatesAsync(CancellationToken cancellationToken)
|
||||
=> this.Client.GetOrchestrationStateAsync(cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the state of selected orchestration instances.
|
||||
/// </summary>
|
||||
/// <returns>List of <see cref="OrchestrationState"/></returns>
|
||||
public Task<IList<OrchestrationState>> GetOrchestrationStateAsync(DateTime? CreatedTimeFrom = default,
|
||||
DateTime? CreatedTimeTo = default,
|
||||
IEnumerable<OrchestrationStatus> RuntimeStatus = default,
|
||||
string InstanceIdPrefix = default,
|
||||
CancellationToken CancellationToken = default)
|
||||
=> this.Client.GetOrchestrationStateAsync(CreatedTimeFrom, CreatedTimeTo, RuntimeStatus, InstanceIdPrefix, CancellationToken);
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Purge history for an orchestration with a specified instance id.
|
||||
/// </summary>
|
||||
/// <param name="instanceId">Instance ID of the orchestration.</param>
|
||||
/// <returns>Class containing number of storage requests sent, along with instances and rows deleted/purged</returns>
|
||||
public Task<int> PurgeInstanceHistoryAsync(string instanceId)
|
||||
=> this.Client.DeleteAllDataForOrchestrationInstance(this.GetPartitionId(instanceId), instanceId);
|
||||
|
||||
/// <summary>
|
||||
/// Purge history for orchestrations that match the specified parameters.
|
||||
/// </summary>
|
||||
/// <param name="createdTimeFrom">CreatedTime of orchestrations. Purges history grater than this value.</param>
|
||||
/// <param name="createdTimeTo">CreatedTime of orchestrations. Purges history less than this value.</param>
|
||||
/// <param name="runtimeStatus">RuntimeStatus of orchestrations. You can specify several status.</param>
|
||||
/// <returns>Class containing number of storage requests sent, along with instances and rows deleted/purged</returns>
|
||||
public Task<int> PurgeInstanceHistoryAsync(DateTime createdTimeFrom, DateTime? createdTimeTo, IEnumerable<OrchestrationStatus> runtimeStatus)
|
||||
=> this.Client.PurgeInstanceHistoryAsync(createdTimeFrom, createdTimeTo, runtimeStatus);
|
||||
|
||||
/// <summary>
|
||||
/// Query orchestration instance states.
|
||||
/// </summary>
|
||||
/// <param name="instanceQuery">The query to perform.</param>
|
||||
/// <param name="pageSize">The page size.</param>
|
||||
/// <param name="continuationToken">The continuation token.</param>
|
||||
/// <param name="cancellationToken">A cancellation token.</param>
|
||||
/// <returns>The result of the query.</returns>
|
||||
public Task<InstanceQueryResult> QueryOrchestrationStatesAsync(InstanceQuery instanceQuery, int pageSize, string continuationToken, CancellationToken cancellationToken)
|
||||
=> this.Client.QueryOrchestrationStatesAsync(instanceQuery, pageSize, continuationToken, cancellationToken);
|
||||
|
||||
/******************************/
|
||||
// Task orchestration methods
|
||||
/******************************/
|
||||
|
||||
async Task<TaskOrchestrationWorkItem> IOrchestrationService.LockNextTaskOrchestrationWorkItemAsync(
|
||||
TimeSpan receiveTimeout,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var nextOrchestrationWorkItem = await this.OrchestrationWorkItemQueue.GetNext(receiveTimeout, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (nextOrchestrationWorkItem != null)
|
||||
{
|
||||
nextOrchestrationWorkItem.MessageBatch.WaitingSince = null;
|
||||
}
|
||||
|
||||
return nextOrchestrationWorkItem;
|
||||
}
|
||||
|
||||
Task IOrchestrationService.CompleteTaskOrchestrationWorkItemAsync(
|
||||
TaskOrchestrationWorkItem workItem,
|
||||
OrchestrationRuntimeState newOrchestrationRuntimeState,
|
||||
IList<TaskMessage> outboundMessages,
|
||||
IList<TaskMessage> orchestratorMessages,
|
||||
IList<TaskMessage> timerMessages,
|
||||
TaskMessage continuedAsNewMessage,
|
||||
OrchestrationState state)
|
||||
{
|
||||
var orchestrationWorkItem = (OrchestrationWorkItem)workItem;
|
||||
var messageBatch = orchestrationWorkItem.MessageBatch;
|
||||
var partition = orchestrationWorkItem.Partition;
|
||||
|
||||
List<TaskMessage> localMessages = null;
|
||||
List<TaskMessage> remoteMessages = null;
|
||||
|
||||
// DurableTask.Core keeps the original runtime state in the work item until after this call returns
|
||||
// but we want it to contain the latest runtime state now (otherwise IsExecutableInstance returns incorrect results)
|
||||
// so we update it now.
|
||||
workItem.OrchestrationRuntimeState = newOrchestrationRuntimeState;
|
||||
|
||||
// all continue as new requests are processed immediately ("fast" continue-as-new)
|
||||
// so by the time we get here, it is not a continue as new
|
||||
partition.Assert(continuedAsNewMessage == null);
|
||||
partition.Assert(workItem.OrchestrationRuntimeState.OrchestrationStatus != OrchestrationStatus.ContinuedAsNew);
|
||||
|
||||
if (orchestratorMessages != null)
|
||||
{
|
||||
foreach (var taskMessage in orchestratorMessages)
|
||||
{
|
||||
if (partition.PartitionId == partition.PartitionFunction(taskMessage.OrchestrationInstance.InstanceId))
|
||||
{
|
||||
if (Entities.IsDelayedEntityMessage(taskMessage, out _))
|
||||
{
|
||||
(timerMessages ?? (timerMessages = new List<TaskMessage>())).Add(taskMessage);
|
||||
}
|
||||
else if (taskMessage.Event is ExecutionStartedEvent executionStartedEvent && executionStartedEvent.ScheduledStartTime.HasValue)
|
||||
{
|
||||
(timerMessages ?? (timerMessages = new List<TaskMessage>())).Add(taskMessage);
|
||||
}
|
||||
else
|
||||
{
|
||||
(localMessages ?? (localMessages = new List<TaskMessage>())).Add(taskMessage);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
(remoteMessages ?? (remoteMessages = new List<TaskMessage>())).Add(taskMessage);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// if this orchestration is not done, and extended sessions are enabled, we keep the work item so we can reuse the execution cursor
|
||||
bool cacheWorkItemForReuse = partition.Settings.ExtendedSessionsEnabled && state.OrchestrationStatus == OrchestrationStatus.Running;
|
||||
|
||||
try
|
||||
{
|
||||
partition.SubmitInternalEvent(new BatchProcessed()
|
||||
{
|
||||
PartitionId = partition.PartitionId,
|
||||
SessionId = messageBatch.SessionId,
|
||||
InstanceId = workItem.InstanceId,
|
||||
BatchStartPosition = messageBatch.BatchStartPosition,
|
||||
BatchLength = messageBatch.BatchLength,
|
||||
NewEvents = (List<HistoryEvent>)newOrchestrationRuntimeState.NewEvents,
|
||||
WorkItemForReuse = cacheWorkItemForReuse ? orchestrationWorkItem : null,
|
||||
State = state,
|
||||
ActivityMessages = (List<TaskMessage>)outboundMessages,
|
||||
LocalMessages = localMessages,
|
||||
RemoteMessages = remoteMessages,
|
||||
TimerMessages = (List<TaskMessage>)timerMessages,
|
||||
Timestamp = DateTime.UtcNow,
|
||||
});
|
||||
}
|
||||
catch(OperationCanceledException e)
|
||||
{
|
||||
// we get here if the partition was terminated. The work is thrown away. It's unavoidable by design, but let's at least create a warning.
|
||||
partition.ErrorHandler.HandleError(nameof(IOrchestrationService.CompleteTaskOrchestrationWorkItemAsync), "Canceling completed orchestration work item because of partition termination", e, false, true);
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
Task IOrchestrationService.AbandonTaskOrchestrationWorkItemAsync(TaskOrchestrationWorkItem workItem)
|
||||
{
|
||||
// We can get here due to transient execution failures of the functions runtime.
|
||||
// In order to guarantee the work is done, we must enqueue a new work item.
|
||||
var orchestrationWorkItem = (OrchestrationWorkItem)workItem;
|
||||
var originalHistorySize = orchestrationWorkItem.OrchestrationRuntimeState.Events.Count - orchestrationWorkItem.OrchestrationRuntimeState.NewEvents.Count;
|
||||
var originalHistory = orchestrationWorkItem.OrchestrationRuntimeState.Events.Take(originalHistorySize).ToList();
|
||||
var newWorkItem = new OrchestrationWorkItem(orchestrationWorkItem.Partition, orchestrationWorkItem.MessageBatch, originalHistory);
|
||||
newWorkItem.Type = OrchestrationWorkItem.ExecutionType.ContinueFromHistory;
|
||||
|
||||
orchestrationWorkItem.Partition.EventTraceHelper.TraceOrchestrationWorkItemQueued(newWorkItem);
|
||||
orchestrationWorkItem.Partition.EnqueueOrchestrationWorkItem(newWorkItem);
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
Task IOrchestrationService.ReleaseTaskOrchestrationWorkItemAsync(TaskOrchestrationWorkItem workItem)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
Task IOrchestrationService.RenewTaskOrchestrationWorkItemLockAsync(TaskOrchestrationWorkItem workItem)
|
||||
{
|
||||
// no renewal required. Work items never time out.
|
||||
return Task.FromResult(workItem);
|
||||
}
|
||||
|
||||
BehaviorOnContinueAsNew IOrchestrationService.EventBehaviourForContinueAsNew
|
||||
=> this.Settings.EventBehaviourForContinueAsNew;
|
||||
|
||||
bool IOrchestrationService.IsMaxMessageCountExceeded(int currentMessageCount, OrchestrationRuntimeState runtimeState)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
int IOrchestrationService.GetDelayInSecondsAfterOnProcessException(Exception exception)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
int IOrchestrationService.GetDelayInSecondsAfterOnFetchException(Exception exception)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
int IOrchestrationService.MaxConcurrentTaskOrchestrationWorkItems => this.Settings.MaxConcurrentOrchestratorFunctions;
|
||||
|
||||
int IOrchestrationService.TaskOrchestrationDispatcherCount => this.Settings.OrchestrationDispatcherCount;
|
||||
|
||||
|
||||
/******************************/
|
||||
// Task activity methods
|
||||
/******************************/
|
||||
|
||||
async Task<TaskActivityWorkItem> IOrchestrationService.LockNextTaskActivityWorkItem(TimeSpan receiveTimeout, CancellationToken cancellationToken)
|
||||
{
|
||||
var nextActivityWorkItem = await this.ActivityWorkItemQueue.GetNext(receiveTimeout, cancellationToken).ConfigureAwait(false);
|
||||
return nextActivityWorkItem;
|
||||
}
|
||||
|
||||
Task IOrchestrationService.AbandonTaskActivityWorkItemAsync(TaskActivityWorkItem workItem)
|
||||
{
|
||||
// put it back into the work queue
|
||||
this.ActivityWorkItemQueue.Add((ActivityWorkItem)workItem);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
Task IOrchestrationService.CompleteTaskActivityWorkItemAsync(TaskActivityWorkItem workItem, TaskMessage responseMessage)
|
||||
{
|
||||
var activityWorkItem = (ActivityWorkItem)workItem;
|
||||
var partition = activityWorkItem.Partition;
|
||||
|
||||
try
|
||||
{
|
||||
partition.SubmitInternalEvent(new ActivityCompleted()
|
||||
{
|
||||
PartitionId = activityWorkItem.Partition.PartitionId,
|
||||
ActivityId = activityWorkItem.ActivityId,
|
||||
OriginPartitionId = activityWorkItem.OriginPartition,
|
||||
ReportedLoad = this.ActivityWorkItemQueue.Load,
|
||||
Timestamp = DateTime.UtcNow,
|
||||
Response = responseMessage,
|
||||
});
|
||||
}
|
||||
catch (OperationCanceledException e)
|
||||
{
|
||||
// we get here if the partition was terminated. The work is thrown away. It's unavoidable by design, but let's at least create a warning.
|
||||
partition.ErrorHandler.HandleError(nameof(IOrchestrationService.CompleteTaskActivityWorkItemAsync), "Canceling completed activity work item because of partition termination", e, false, true);
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
Task<TaskActivityWorkItem> IOrchestrationService.RenewTaskActivityWorkItemLockAsync(TaskActivityWorkItem workItem)
|
||||
{
|
||||
// no renewal required. Work items never time out.
|
||||
return Task.FromResult(workItem);
|
||||
}
|
||||
|
||||
int IOrchestrationService.MaxConcurrentTaskActivityWorkItems => this.Settings.MaxConcurrentActivityFunctions;
|
||||
|
||||
int IOrchestrationService.TaskActivityDispatcherCount => this.Settings.ActivityDispatcherCount;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,258 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using DurableTask.Core;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Newtonsoft.Json;
|
||||
using Newtonsoft.Json.Converters;
|
||||
|
||||
/// <summary>
|
||||
/// Settings for the <see cref="NetheriteOrchestrationService"/> class.
|
||||
/// </summary>
|
||||
[JsonObject]
|
||||
public class NetheriteOrchestrationServiceSettings
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the connection string for the event hubs namespace, if needed.
|
||||
/// </summary>
|
||||
public string EventHubsConnectionString { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the EventProcessor management
|
||||
/// </summary>
|
||||
public string EventProcessorManagement { get; set; } = "EventHubs";
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the connection string for the Azure storage account, supporting all types of blobs, and table storage.
|
||||
/// </summary>
|
||||
public string StorageConnectionString { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the connection string for a premium Azure storage account supporting page blobs only.
|
||||
/// </summary>
|
||||
public string PremiumStorageConnectionString { get; set; }
|
||||
|
||||
[JsonIgnore]
|
||||
internal bool UsePremiumStorage => !string.IsNullOrEmpty(this.PremiumStorageConnectionString);
|
||||
|
||||
/// <summary>
|
||||
/// The name of the taskhub. Matches Microsoft.Azure.WebJobs.Extensions.DurableTask.
|
||||
/// </summary>
|
||||
public string HubName { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the identifier for the current worker.
|
||||
/// </summary>
|
||||
public string WorkerId { get; set; } = Environment.MachineName;
|
||||
|
||||
/// <summary>
|
||||
/// The name to use for the Azure table with the load information
|
||||
/// </summary>
|
||||
public string LoadInformationAzureTableName { get; set; } = "DurableTaskPartitions";
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the maximum number of work items that can be processed concurrently on a single node.
|
||||
/// The default value is 100.
|
||||
/// Matches Microsoft.Azure.WebJobs.Extensions.DurableTask.
|
||||
/// </summary>
|
||||
public int MaxConcurrentActivityFunctions { get; set; } = 100;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the maximum number of orchestrations that can be processed concurrently on a single node.
|
||||
/// The default value is 100.
|
||||
/// Matches Microsoft.Azure.WebJobs.Extensions.DurableTask.
|
||||
/// </summary>
|
||||
public int MaxConcurrentOrchestratorFunctions { get; set; } = 100;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the number of dispatchers used to dispatch orchestrations.
|
||||
/// </summary>
|
||||
public int OrchestrationDispatcherCount { get; set; } = 1;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the number of dispatchers used to dispatch activities.
|
||||
/// </summary>
|
||||
public int ActivityDispatcherCount { get; set; } = 1;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets a flag indicating whether to enable caching of execution cursors to avoid replay.
|
||||
/// Matches Microsoft.Azure.WebJobs.Extensions.DurableTask.
|
||||
/// </summary>
|
||||
public bool ExtendedSessionsEnabled { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Whether we should carry over unexecuted raised events to the next iteration of an orchestration on ContinueAsNew.
|
||||
/// </summary>
|
||||
public BehaviorOnContinueAsNew EventBehaviourForContinueAsNew { get; set; } = BehaviorOnContinueAsNew.Carryover;
|
||||
|
||||
/// <summary>
|
||||
/// When true, will throw an exception when attempting to create an orchestration with an existing dedupe status.
|
||||
/// </summary>
|
||||
public bool ThrowExceptionOnInvalidDedupeStatus { get; set; } = false;
|
||||
|
||||
/// <summary>
|
||||
/// Whether to keep the orchestration service running even if stop is called.
|
||||
/// This is useful in a testing scenario, due to the inordinate time spent when shutting down EventProcessorHost.
|
||||
/// </summary>
|
||||
public bool KeepServiceRunning { get; set; } = false;
|
||||
|
||||
/// <summary>
|
||||
/// Whether to checkpoint the current state of a partition when it is stopped. This improves recovery time but
|
||||
/// lengthens shutdown time.
|
||||
/// </summary>
|
||||
public bool TakeStateCheckpointWhenStoppingPartition { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// A limit on how many bytes to append to the log before initiating a state checkpoint. The default is 20MB.
|
||||
/// </summary>
|
||||
public long MaxNumberBytesBetweenCheckpoints { get; set; } = 20 * 1024 * 1024;
|
||||
|
||||
/// <summary>
|
||||
/// A limit on how many events to append to the log before initiating a state checkpoint. The default is 10000.
|
||||
/// </summary>
|
||||
public long MaxNumberEventsBetweenCheckpoints { get; set; } = 10 * 1000;
|
||||
|
||||
/// <summary>
|
||||
/// A limit on how long to wait between state checkpoints, in milliseconds. The default is 60s.
|
||||
/// </summary>
|
||||
public long MaxTimeMsBetweenCheckpoints { get; set; } = 60 * 1000;
|
||||
|
||||
/// <summary>
|
||||
/// Whether to use the Faster PSF support for handling queries.
|
||||
/// </summary>
|
||||
public bool UsePSFQueries { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Whether to use the alternate object store implementation.
|
||||
/// </summary>
|
||||
public bool UseAlternateObjectStore { get; set; } = false;
|
||||
|
||||
/// <summary>
|
||||
/// Forces steps to pe persisted before applying their effects, thus disabling all speculation.
|
||||
/// </summary>
|
||||
public bool PersistStepsFirst { get; set; } = false;
|
||||
|
||||
/// <summary>
|
||||
/// Whether to use JSON serialization for eventhubs packets.
|
||||
/// </summary>
|
||||
[JsonConverter(typeof(StringEnumConverter))]
|
||||
public JsonPacketUse UseJsonPackets { get; set; } = JsonPacketUse.Never;
|
||||
|
||||
/// <summary>
|
||||
/// Which packets to send in JSON format.
|
||||
/// </summary>
|
||||
public enum JsonPacketUse
|
||||
{
|
||||
/// <summary>
|
||||
/// Never send packets in JSON format
|
||||
/// </summary>
|
||||
Never,
|
||||
|
||||
/// <summary>
|
||||
/// Send packets to clients in JSON format
|
||||
/// </summary>
|
||||
ForClients,
|
||||
|
||||
/// <summary>
|
||||
/// Send all packets in JSON format
|
||||
/// </summary>
|
||||
ForAll,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A lower limit on the severity level of trace events emitted by the transport layer.
|
||||
/// </summary>
|
||||
/// <remarks>This level applies to both ETW events and ILogger events.</remarks>
|
||||
[JsonConverter(typeof(StringEnumConverter))]
|
||||
public LogLevel TransportLogLevelLimit { get; set; } = LogLevel.Information;
|
||||
|
||||
/// <summary>
|
||||
/// A lower limit on the severity level of trace events emitted by the storage layer.
|
||||
/// </summary>
|
||||
/// <remarks>This level applies to both ETW events and ILogger events.</remarks>
|
||||
[JsonConverter(typeof(StringEnumConverter))]
|
||||
public LogLevel StorageLogLevelLimit { get; set; } = LogLevel.Information;
|
||||
|
||||
/// <summary>
|
||||
/// A lower limit on the severity level of event processor trace events emitted.
|
||||
/// </summary>
|
||||
/// <remarks>This level applies to both ETW events and ILogger events.</remarks>
|
||||
[JsonConverter(typeof(StringEnumConverter))]
|
||||
public LogLevel EventLogLevelLimit { get; set; } = LogLevel.Warning;
|
||||
|
||||
/// <summary>
|
||||
/// A lower limit on the severity level of all other trace events emitted.
|
||||
/// </summary>
|
||||
/// <remarks>This level applies to both ETW events and ILogger events.</remarks>
|
||||
[JsonConverter(typeof(StringEnumConverter))]
|
||||
public LogLevel LogLevelLimit { get; set; } = LogLevel.Information;
|
||||
|
||||
/// <summary>
|
||||
/// Validates the specified <see cref="NetheriteOrchestrationServiceSettings"/> object.
|
||||
/// </summary>
|
||||
/// <param name="settings">The <see cref="NetheriteOrchestrationServiceSettings"/> object to validate.</param>
|
||||
/// <returns>Returns <paramref name="settings"/> if successfully validated.</returns>
|
||||
public static NetheriteOrchestrationServiceSettings Validate(NetheriteOrchestrationServiceSettings settings)
|
||||
{
|
||||
if (settings == null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(settings));
|
||||
}
|
||||
|
||||
if (string.IsNullOrEmpty(settings.EventHubsConnectionString))
|
||||
{
|
||||
throw new ArgumentNullException(nameof(settings.EventHubsConnectionString));
|
||||
}
|
||||
|
||||
TransportConnectionString.Parse(settings.EventHubsConnectionString, out var storage, out var transport, out int? numPartitions);
|
||||
|
||||
if (storage != TransportConnectionString.StorageChoices.Memory || transport != TransportConnectionString.TransportChoices.Memory)
|
||||
{
|
||||
if (string.IsNullOrEmpty(settings.StorageConnectionString))
|
||||
{
|
||||
throw new ArgumentNullException(nameof(settings.StorageConnectionString));
|
||||
}
|
||||
}
|
||||
|
||||
if ((transport != TransportConnectionString.TransportChoices.EventHubs))
|
||||
{
|
||||
if (numPartitions < 1 || numPartitions > 32)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(nameof(settings.EventHubsConnectionString));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (string.IsNullOrEmpty(TransportConnectionString.EventHubsNamespaceName(settings.EventHubsConnectionString)))
|
||||
{
|
||||
throw new FormatException(nameof(settings.EventHubsConnectionString));
|
||||
}
|
||||
}
|
||||
|
||||
if (settings.MaxConcurrentOrchestratorFunctions <= 0)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(nameof(settings.MaxConcurrentOrchestratorFunctions));
|
||||
}
|
||||
|
||||
if (settings.MaxConcurrentActivityFunctions <= 0)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(nameof(settings.MaxConcurrentActivityFunctions));
|
||||
}
|
||||
|
||||
return settings;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,164 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using DurableTask.Core;
|
||||
using DurableTask.Core.History;
|
||||
|
||||
class OrchestrationMessageBatch : InternalReadEvent
|
||||
{
|
||||
public string InstanceId;
|
||||
public long SessionId;
|
||||
public long BatchStartPosition;
|
||||
public int BatchLength;
|
||||
public bool ForceNewExecution;
|
||||
public List<TaskMessage> MessagesToProcess;
|
||||
public string WorkItemId;
|
||||
public double? WaitingSince; // measures time waiting to execute
|
||||
|
||||
public override EventId EventId => EventId.MakePartitionInternalEventId(this.WorkItemId);
|
||||
|
||||
public OrchestrationMessageBatch(string instanceId, SessionsState.Session session, Partition partition)
|
||||
{
|
||||
this.InstanceId = instanceId;
|
||||
this.SessionId = session.SessionId;
|
||||
this.BatchStartPosition = session.BatchStartPosition;
|
||||
this.BatchLength = session.Batch.Count;
|
||||
this.ForceNewExecution = session.ForceNewExecution && session.BatchStartPosition == 0;
|
||||
this.MessagesToProcess = session.Batch.ToList(); // make a copy, because it can be modified when new messages arrive
|
||||
this.WorkItemId = SessionsState.GetWorkItemId(partition.PartitionId, this.SessionId, this.BatchStartPosition);
|
||||
this.WaitingSince = partition.CurrentTimeMs;
|
||||
|
||||
session.CurrentBatch = this;
|
||||
|
||||
partition.EventDetailTracer?.TraceEventProcessingDetail($"OrchestrationMessageBatch is prefetching instance={this.InstanceId} batch={this.WorkItemId}");
|
||||
|
||||
// continue when we have the history state loaded, which gives us the latest state and/or cursor
|
||||
partition.SubmitInternalEvent(this);
|
||||
}
|
||||
|
||||
public override TrackedObjectKey ReadTarget => TrackedObjectKey.History(this.InstanceId);
|
||||
|
||||
public override TrackedObjectKey? Prefetch => TrackedObjectKey.Instance(this.InstanceId);
|
||||
|
||||
public double WaitTimeMs(double now) => (now - this.WaitingSince) ?? 0;
|
||||
|
||||
public override void OnReadComplete(TrackedObject s, Partition partition)
|
||||
{
|
||||
var historyState = (HistoryState)s;
|
||||
OrchestrationWorkItem workItem;
|
||||
|
||||
if (this.ForceNewExecution || historyState == null)
|
||||
{
|
||||
// we either have no previous instance, or want to replace the previous instance
|
||||
workItem = new OrchestrationWorkItem(partition, this, previousHistory: null);
|
||||
workItem.Type = OrchestrationWorkItem.ExecutionType.Fresh;
|
||||
}
|
||||
else if (historyState.CachedOrchestrationWorkItem != null)
|
||||
{
|
||||
// we have a cached cursor and can resume where we left off
|
||||
workItem = historyState.CachedOrchestrationWorkItem;
|
||||
workItem.SetNextMessageBatch(this);
|
||||
workItem.Type = OrchestrationWorkItem.ExecutionType.ContinueFromCursor;
|
||||
|
||||
// sanity check: it appears cursor is sometimes corrupted, in that case, construct fresh
|
||||
// TODO investigate reasons and fix root cause
|
||||
if (workItem.OrchestrationRuntimeState?.Events?.Count != historyState.History?.Count
|
||||
|| workItem.OrchestrationRuntimeState?.OrchestrationInstance?.ExecutionId != historyState.ExecutionId)
|
||||
{
|
||||
partition.EventTraceHelper.TraceWarning($"Fixing bad workitem cache instance={this.InstanceId} batch={this.WorkItemId} expected_size={historyState.History?.Count} actual_size={workItem.OrchestrationRuntimeState?.Events?.Count} expected_executionid={historyState.ExecutionId} actual_executionid={workItem.OrchestrationRuntimeState?.OrchestrationInstance?.ExecutionId}");
|
||||
|
||||
// we create a new work item and rehydrate the instance from its history
|
||||
workItem = new OrchestrationWorkItem(partition, this, previousHistory: historyState.History);
|
||||
workItem.Type = OrchestrationWorkItem.ExecutionType.ContinueFromHistory;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// we have to rehydrate the instance from its history
|
||||
workItem = new OrchestrationWorkItem(partition, this, previousHistory: historyState.History);
|
||||
workItem.Type = OrchestrationWorkItem.ExecutionType.ContinueFromHistory;
|
||||
}
|
||||
|
||||
if (!this.IsExecutableInstance(workItem, out var reason))
|
||||
{
|
||||
// this instance cannot be executed, so we are discarding the messages
|
||||
foreach (var taskMessage in workItem.MessageBatch.MessagesToProcess)
|
||||
{
|
||||
partition.EventTraceHelper.TraceTaskMessageDiscarded(taskMessage, reason, workItem.MessageBatch.WorkItemId);
|
||||
}
|
||||
|
||||
// we issue a batch processed event which will remove the messages without updating the instance state
|
||||
partition.SubmitInternalEvent(new BatchProcessed()
|
||||
{
|
||||
PartitionId = partition.PartitionId,
|
||||
SessionId = this.SessionId,
|
||||
InstanceId = this.InstanceId,
|
||||
BatchStartPosition = this.BatchStartPosition,
|
||||
BatchLength = this.BatchLength,
|
||||
NewEvents = null,
|
||||
State = null,
|
||||
WorkItemForReuse = null,
|
||||
ActivityMessages = null,
|
||||
LocalMessages = null,
|
||||
RemoteMessages = null,
|
||||
TimerMessages = null,
|
||||
Timestamp = DateTime.UtcNow,
|
||||
});
|
||||
}
|
||||
else
|
||||
{
|
||||
partition.EventTraceHelper.TraceOrchestrationWorkItemQueued(workItem);
|
||||
|
||||
// the work item is ready to process - put it into the OrchestrationWorkItemQueue
|
||||
partition.EnqueueOrchestrationWorkItem(workItem);
|
||||
}
|
||||
}
|
||||
|
||||
bool IsExecutableInstance(TaskOrchestrationWorkItem workItem, out string message)
|
||||
{
|
||||
if (workItem.OrchestrationRuntimeState.ExecutionStartedEvent == null
|
||||
&& !this.MessagesToProcess.Any(msg => msg.Event is ExecutionStartedEvent))
|
||||
{
|
||||
if (DurableTask.Core.Common.Entities.AutoStart(this.InstanceId, this.MessagesToProcess))
|
||||
{
|
||||
message = default;
|
||||
return true;
|
||||
}
|
||||
else
|
||||
{
|
||||
message = workItem.NewMessages.Count == 0 ? "No such instance" : "Instance is corrupted";
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (workItem.OrchestrationRuntimeState.ExecutionStartedEvent != null &&
|
||||
workItem.OrchestrationRuntimeState.OrchestrationStatus != OrchestrationStatus.Running &&
|
||||
workItem.OrchestrationRuntimeState.OrchestrationStatus != OrchestrationStatus.Pending)
|
||||
{
|
||||
message = $"Instance is {workItem.OrchestrationRuntimeState.OrchestrationStatus}";
|
||||
return false;
|
||||
}
|
||||
|
||||
message = null;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using DurableTask.Core;
|
||||
using DurableTask.Core.History;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
class OrchestrationWorkItem : TaskOrchestrationWorkItem
|
||||
{
|
||||
public OrchestrationMessageBatch MessageBatch { get; set; }
|
||||
|
||||
public Partition Partition { get; }
|
||||
|
||||
public ExecutionType Type { get; set; }
|
||||
|
||||
public enum ExecutionType { Fresh, ContinueFromHistory, ContinueFromCursor };
|
||||
|
||||
public OrchestrationWorkItem(Partition partition, OrchestrationMessageBatch messageBatch, List<HistoryEvent> previousHistory = null)
|
||||
{
|
||||
this.Partition = partition;
|
||||
this.MessageBatch = messageBatch;
|
||||
this.InstanceId = messageBatch.InstanceId;
|
||||
this.NewMessages = messageBatch.MessagesToProcess;
|
||||
this.OrchestrationRuntimeState = new OrchestrationRuntimeState(previousHistory);
|
||||
this.LockedUntilUtc = DateTime.MaxValue; // this backend does not require workitem lock renewals
|
||||
this.Session = null; // we don't use the extended session API because we are caching cursors in the work item
|
||||
}
|
||||
|
||||
public void SetNextMessageBatch(OrchestrationMessageBatch messageBatch)
|
||||
{
|
||||
this.MessageBatch = messageBatch;
|
||||
this.NewMessages = messageBatch.MessagesToProcess;
|
||||
this.OrchestrationRuntimeState.NewEvents.Clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,285 @@
|
|||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using DurableTask.Core;
|
||||
using DurableTask.Core.Common;
|
||||
using DurableTask.Core.History;
|
||||
using DurableTask.Core.Tracing;
|
||||
using DurableTask.Netherite.Scaling;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
partial class Partition : TransportAbstraction.IPartition
|
||||
{
|
||||
readonly NetheriteOrchestrationService host;
|
||||
readonly Stopwatch stopwatch = new Stopwatch();
|
||||
|
||||
public uint PartitionId { get; private set; }
|
||||
public string TracePrefix { get; private set; }
|
||||
public Func<string, uint> PartitionFunction { get; private set; }
|
||||
public Func<uint> NumberPartitions { get; private set; }
|
||||
public IPartitionErrorHandler ErrorHandler { get; private set; }
|
||||
public PartitionTraceHelper TraceHelper { get; private set; }
|
||||
|
||||
public double CurrentTimeMs => this.stopwatch.Elapsed.TotalMilliseconds;
|
||||
|
||||
public NetheriteOrchestrationServiceSettings Settings { get; private set; }
|
||||
public string StorageAccountName { get; private set; }
|
||||
|
||||
public IPartitionState State { get; private set; }
|
||||
public TransportAbstraction.ISender BatchSender { get; private set; }
|
||||
public WorkItemQueue<ActivityWorkItem> ActivityWorkItemQueue { get; private set; }
|
||||
public WorkItemQueue<OrchestrationWorkItem> OrchestrationWorkItemQueue { get; private set; }
|
||||
public LoadPublisher LoadPublisher { get; private set; }
|
||||
|
||||
public BatchTimer<PartitionEvent> PendingTimers { get; private set; }
|
||||
|
||||
public EventTraceHelper EventTraceHelper { get; }
|
||||
|
||||
public bool RecoveryIsComplete { get; private set; }
|
||||
|
||||
// A little helper property that allows us to conventiently check the condition for low-level event tracing
|
||||
public EventTraceHelper EventDetailTracer => this.EventTraceHelper.IsTracingAtMostDetailedLevel ? this.EventTraceHelper : null;
|
||||
|
||||
static readonly SemaphoreSlim MaxConcurrentStarts = new SemaphoreSlim(5);
|
||||
|
||||
public Partition(
|
||||
NetheriteOrchestrationService host,
|
||||
uint partitionId,
|
||||
Func<string, uint> partitionFunction,
|
||||
Func<uint> numberPartitions,
|
||||
TransportAbstraction.ISender batchSender,
|
||||
NetheriteOrchestrationServiceSettings settings,
|
||||
string storageAccountName,
|
||||
WorkItemQueue<ActivityWorkItem> activityWorkItemQueue,
|
||||
WorkItemQueue<OrchestrationWorkItem> orchestrationWorkItemQueue,
|
||||
LoadPublisher loadPublisher)
|
||||
{
|
||||
this.host = host;
|
||||
this.PartitionId = partitionId;
|
||||
this.PartitionFunction = partitionFunction;
|
||||
this.NumberPartitions = numberPartitions;
|
||||
this.BatchSender = batchSender;
|
||||
this.Settings = settings;
|
||||
this.StorageAccountName = storageAccountName;
|
||||
this.ActivityWorkItemQueue = activityWorkItemQueue;
|
||||
this.OrchestrationWorkItemQueue = orchestrationWorkItemQueue;
|
||||
this.LoadPublisher = loadPublisher;
|
||||
this.TraceHelper = new PartitionTraceHelper(host.Logger, settings.LogLevelLimit, this.StorageAccountName, this.Settings.HubName, this.PartitionId);
|
||||
this.EventTraceHelper = new EventTraceHelper(host.LoggerFactory, settings.EventLogLevelLimit, this);
|
||||
this.stopwatch.Start();
|
||||
}
|
||||
|
||||
public async Task<long> CreateOrRestoreAsync(IPartitionErrorHandler errorHandler, long firstInputQueuePosition)
|
||||
{
|
||||
EventTraceContext.Clear();
|
||||
|
||||
this.ErrorHandler = errorHandler;
|
||||
this.TraceHelper.TraceProgress("Starting partition");
|
||||
|
||||
await MaxConcurrentStarts.WaitAsync();
|
||||
|
||||
// create or restore partition state from last snapshot
|
||||
try
|
||||
{
|
||||
// create the state
|
||||
this.State = ((IStorageProvider)this.host).CreatePartitionState();
|
||||
|
||||
// initialize timer for this partition
|
||||
this.PendingTimers = new BatchTimer<PartitionEvent>(this.ErrorHandler.Token, this.TimersFired);
|
||||
|
||||
// goes to storage to create or restore the partition state
|
||||
this.TraceHelper.TraceProgress("Loading partition state");
|
||||
var inputQueuePosition = await this.State.CreateOrRestoreAsync(this, this.ErrorHandler, firstInputQueuePosition).ConfigureAwait(false);
|
||||
|
||||
this.RecoveryIsComplete = true;
|
||||
|
||||
this.PendingTimers.Start($"Timer{this.PartitionId:D2}");
|
||||
|
||||
this.TraceHelper.TraceProgress($"Started partition, nextInputQueuePosition={inputQueuePosition}");
|
||||
return inputQueuePosition;
|
||||
}
|
||||
catch (Exception e) when (!Utils.IsFatal(e))
|
||||
{
|
||||
this.ErrorHandler.HandleError(nameof(CreateOrRestoreAsync), "Could not start partition", e, true, false);
|
||||
throw;
|
||||
}
|
||||
finally
|
||||
{
|
||||
MaxConcurrentStarts.Release();
|
||||
}
|
||||
}
|
||||
|
||||
[Conditional("DEBUG")]
|
||||
public void Assert(bool condition)
|
||||
{
|
||||
if (!condition)
|
||||
{
|
||||
if (System.Diagnostics.Debugger.IsAttached)
|
||||
{
|
||||
System.Diagnostics.Debugger.Break();
|
||||
}
|
||||
|
||||
var stacktrace = System.Environment.StackTrace;
|
||||
|
||||
this.ErrorHandler.HandleError(stacktrace, "Assertion failed", null, false, false);
|
||||
}
|
||||
}
|
||||
|
||||
public async Task StopAsync(bool isForced)
|
||||
{
|
||||
this.TraceHelper.TraceProgress("Stopping partition");
|
||||
|
||||
try
|
||||
{
|
||||
if (!this.ErrorHandler.IsTerminated)
|
||||
{
|
||||
bool takeCheckpoint = this.Settings.TakeStateCheckpointWhenStoppingPartition && !isForced;
|
||||
// for a clean shutdown we try to save some of the latest progress to storage and then release the lease
|
||||
await this.State.CleanShutdown(takeCheckpoint).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
catch(OperationCanceledException) when (this.ErrorHandler.IsTerminated)
|
||||
{
|
||||
// o.k. during termination
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
this.ErrorHandler.HandleError(nameof(StopAsync), "Could not shut down partition state cleanly", e, true, false);
|
||||
}
|
||||
|
||||
// at this point, the partition has been terminated (either cleanly or by exception)
|
||||
this.Assert(this.ErrorHandler.IsTerminated);
|
||||
|
||||
// tell the load publisher to send all buffered info
|
||||
this.LoadPublisher?.Flush();
|
||||
|
||||
this.TraceHelper.TraceProgress("Stopped partition");
|
||||
}
|
||||
|
||||
void TimersFired(List<PartitionEvent> timersFired)
|
||||
{
|
||||
try
|
||||
{
|
||||
foreach (var t in timersFired)
|
||||
{
|
||||
switch(t)
|
||||
{
|
||||
case PartitionUpdateEvent updateEvent:
|
||||
this.SubmitInternalEvent(updateEvent);
|
||||
break;
|
||||
case PartitionReadEvent readEvent:
|
||||
this.SubmitInternalEvent(readEvent);
|
||||
break;
|
||||
case PartitionQueryEvent queryEvent:
|
||||
this.SubmitInternalEvent(queryEvent);
|
||||
break;
|
||||
default:
|
||||
throw new InvalidCastException("Could not cast to neither PartitionUpdateEvent nor PartitionReadEvent");
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (this.ErrorHandler.IsTerminated)
|
||||
{
|
||||
// o.k. during termination
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
this.ErrorHandler.HandleError("TimersFired", "Encountered exception while firing partition timers", e, true, false);
|
||||
}
|
||||
}
|
||||
|
||||
public void Send(ClientEvent clientEvent)
|
||||
{
|
||||
this.EventDetailTracer?.TraceEventProcessingDetail($"Sending client event {clientEvent} id={clientEvent.EventId}");
|
||||
|
||||
this.BatchSender.Submit(clientEvent);
|
||||
}
|
||||
|
||||
public void Send(PartitionUpdateEvent updateEvent)
|
||||
{
|
||||
this.EventDetailTracer?.TraceEventProcessingDetail($"Sending partition update event {updateEvent} id={updateEvent.EventId}");
|
||||
|
||||
// trace DTFx TaskMessages that are sent to other participants
|
||||
if (this.RecoveryIsComplete)
|
||||
{
|
||||
foreach (var taskMessage in updateEvent.TracedTaskMessages)
|
||||
{
|
||||
this.EventTraceHelper.TraceTaskMessageSent(taskMessage, updateEvent.EventIdString);
|
||||
}
|
||||
}
|
||||
|
||||
this.BatchSender.Submit(updateEvent);
|
||||
}
|
||||
|
||||
public void SubmitInternalEvent(PartitionUpdateEvent updateEvent)
|
||||
{
|
||||
// for better analytics experience, trace DTFx TaskMessages that are "sent"
|
||||
// by this partition to itself the same way as if sent to other partitions
|
||||
if (this.RecoveryIsComplete)
|
||||
{
|
||||
foreach (var taskMessage in updateEvent.TracedTaskMessages)
|
||||
{
|
||||
this.EventTraceHelper.TraceTaskMessageSent(taskMessage, updateEvent.EventIdString);
|
||||
}
|
||||
}
|
||||
|
||||
updateEvent.ReceivedTimestamp = this.CurrentTimeMs;
|
||||
|
||||
this.State.SubmitInternalEvent(updateEvent);
|
||||
}
|
||||
|
||||
public void SubmitInternalEvent(PartitionReadEvent readEvent)
|
||||
{
|
||||
readEvent.ReceivedTimestamp = this.CurrentTimeMs;
|
||||
this.State.SubmitInternalEvent(readEvent);
|
||||
}
|
||||
|
||||
public void SubmitInternalEvent(PartitionQueryEvent queryEvent)
|
||||
{
|
||||
queryEvent.ReceivedTimestamp = this.CurrentTimeMs;
|
||||
this.State.SubmitInternalEvent(queryEvent);
|
||||
}
|
||||
|
||||
public void SubmitExternalEvents(IList<PartitionEvent> partitionEvents)
|
||||
{
|
||||
foreach (PartitionEvent partitionEvent in partitionEvents)
|
||||
{
|
||||
partitionEvent.ReceivedTimestamp = this.CurrentTimeMs;
|
||||
}
|
||||
|
||||
this.State.SubmitExternalEvents(partitionEvents);
|
||||
}
|
||||
|
||||
public void EnqueueActivityWorkItem(ActivityWorkItem item)
|
||||
{
|
||||
this.EventDetailTracer?.TraceEventProcessingDetail($"Enqueueing ActivityWorkItem {item.WorkItemId}");
|
||||
this.ActivityWorkItemQueue.Add(item);
|
||||
}
|
||||
|
||||
public void EnqueueOrchestrationWorkItem(OrchestrationWorkItem item)
|
||||
{
|
||||
this.EventDetailTracer?.TraceEventProcessingDetail($"Enqueueing OrchestrationWorkItem {item.MessageBatch.WorkItemId}");
|
||||
this.OrchestrationWorkItemQueue.Add(item);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,98 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using System.Threading;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
// For indicating and initiating termination, and for tracing errors and warnings relating to a partition.
|
||||
// Is is basically a wrapper around CancellationTokenSource with features for diagnostics.
|
||||
class PartitionErrorHandler : IPartitionErrorHandler
|
||||
{
|
||||
readonly CancellationTokenSource cts = new CancellationTokenSource();
|
||||
readonly int partitionId;
|
||||
readonly ILogger logger;
|
||||
readonly LogLevel logLevelLimit;
|
||||
readonly string account;
|
||||
readonly string taskHub;
|
||||
|
||||
public CancellationToken Token => this.cts.Token;
|
||||
public bool IsTerminated => this.cts.Token.IsCancellationRequested;
|
||||
|
||||
public PartitionErrorHandler(int partitionId, ILogger logger, LogLevel logLevelLimit, string storageAccountName, string taskHubName)
|
||||
{
|
||||
this.cts = new CancellationTokenSource();
|
||||
this.partitionId = partitionId;
|
||||
this.logger = logger;
|
||||
this.logLevelLimit = logLevelLimit;
|
||||
this.account = storageAccountName;
|
||||
this.taskHub = taskHubName;
|
||||
}
|
||||
|
||||
public void HandleError(string context, string message, Exception exception, bool terminatePartition, bool isWarning)
|
||||
{
|
||||
this.TraceError(isWarning, context, message, exception, terminatePartition);
|
||||
|
||||
// terminate this partition in response to the error
|
||||
if (terminatePartition && !this.cts.IsCancellationRequested)
|
||||
{
|
||||
this.Terminate();
|
||||
}
|
||||
}
|
||||
|
||||
void TraceError(bool isWarning, string context, string message, Exception exception, bool terminatePartition)
|
||||
{
|
||||
var logLevel = isWarning ? LogLevel.Warning : LogLevel.Error;
|
||||
if (this.logLevelLimit <= logLevel)
|
||||
{
|
||||
this.logger?.Log(logLevel, "Part{partition:D2} !!! {message} in {context}: {exception} terminatePartition={terminatePartition}", this.partitionId, message, context, exception, terminatePartition);
|
||||
|
||||
if (isWarning)
|
||||
{
|
||||
EtwSource.Log.PartitionWarning(this.account, this.taskHub, this.partitionId, context, terminatePartition, message, exception?.ToString() ?? string.Empty, TraceUtils.ExtensionVersion);
|
||||
}
|
||||
else
|
||||
{
|
||||
EtwSource.Log.PartitionError(this.account, this.taskHub, this.partitionId, context, terminatePartition, message, exception?.ToString() ?? string.Empty, TraceUtils.ExtensionVersion);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void TerminateNormally()
|
||||
{
|
||||
this.Terminate();
|
||||
}
|
||||
|
||||
void Terminate()
|
||||
{
|
||||
try
|
||||
{
|
||||
this.cts.Cancel();
|
||||
}
|
||||
catch (AggregateException aggregate)
|
||||
{
|
||||
foreach (var e in aggregate.InnerExceptions)
|
||||
{
|
||||
this.HandleError("PartitionErrorHandler.Terminate", "Encountered exeption while canceling token", e, false, true);
|
||||
}
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
this.HandleError("PartitionErrorHandler.Terminate", "Encountered exeption while canceling token", e, false, true);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,82 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
using DurableTask.Core;
|
||||
using DurableTask.Netherite.Scaling;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
class PartitionTraceHelper
|
||||
{
|
||||
readonly ILogger logger;
|
||||
readonly string account;
|
||||
readonly string taskHub;
|
||||
readonly int partitionId;
|
||||
readonly LogLevel logLevelLimit;
|
||||
|
||||
public PartitionTraceHelper(ILogger logger, LogLevel logLevelLimit, string storageAccountName, string taskHubName, uint partitionId)
|
||||
{
|
||||
this.logger = logger;
|
||||
this.account = storageAccountName;
|
||||
this.taskHub = taskHubName;
|
||||
this.partitionId = (int)partitionId;
|
||||
this.logLevelLimit = logLevelLimit;
|
||||
}
|
||||
|
||||
public void TraceProgress(string details)
|
||||
{
|
||||
if (this.logLevelLimit <= LogLevel.Information)
|
||||
{
|
||||
this.logger.LogInformation("Part{partition:D2} {details}", this.partitionId, details);
|
||||
if (EtwSource.Log.IsEnabled())
|
||||
{
|
||||
EtwSource.Log.PartitionProgress(this.account, this.taskHub, this.partitionId, details, TraceUtils.ExtensionVersion);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void TracePartitionLoad(PartitionLoadInfo info)
|
||||
{
|
||||
if (this.logLevelLimit <= LogLevel.Information)
|
||||
{
|
||||
this.logger.LogInformation("Part{partition:D2} Publishing LoadInfo WorkItems={workItems} Activities={activities} Timers={timers} Requests={requests} Outbox={outbox} Wakeup={wakeup} ActivityLatencyMs={activityLatencyMs} WorkItemLatencyMs={workItemLatencyMs} WorkerId={workerId} LatencyTrend={latencyTrend} MissRate={missRate} InputQueuePosition={inputQueuePosition} CommitLogPosition={commitLogPosition}",
|
||||
this.partitionId, info.WorkItems, info.Activities, info.Timers, info.Requests, info.Outbox, info.Wakeup, info.ActivityLatencyMs, info.WorkItemLatencyMs, info.WorkerId, info.LatencyTrend, info.MissRate, info.InputQueuePosition, info.CommitLogPosition);
|
||||
|
||||
if (EtwSource.Log.IsEnabled())
|
||||
{
|
||||
EtwSource.Log.PartitionLoadPublished(this.account, this.taskHub, this.partitionId, info.WorkItems, info.Activities, info.Timers, info.Requests, info.Outbox, info.Wakeup?.ToString("o") ?? "", info.ActivityLatencyMs, info.WorkItemLatencyMs, info.WorkerId, info.LatencyTrend, info.MissRate, info.InputQueuePosition, info.CommitLogPosition, TraceUtils.ExtensionVersion);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void TraceWorkItemProgress(string workItemId, string instanceId, string format, params object[] args)
|
||||
{
|
||||
if (this.logLevelLimit <= LogLevel.Debug)
|
||||
{
|
||||
if (this.logger.IsEnabled(LogLevel.Debug))
|
||||
{
|
||||
object[] objarray = new object[3 + args.Length];
|
||||
objarray[0] = this.partitionId;
|
||||
objarray[1] = workItemId;
|
||||
objarray[2] = instanceId;
|
||||
Array.Copy(args, 0, objarray, 3, args.Length);
|
||||
this.logger.LogDebug("Part{partition:D2} OrchestrationWorkItem workItemId={workItemId} instanceId={instanceId} " + format, objarray);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,90 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
using Microsoft.Azure.EventHubs;
|
||||
|
||||
/// <summary>
|
||||
/// Encapsulates how the transport connection string setting is interpreted.
|
||||
/// </summary>
|
||||
public static class TransportConnectionString
|
||||
{
|
||||
/// <summary>
|
||||
/// Configuration options for the storage component
|
||||
/// </summary>
|
||||
public enum StorageChoices
|
||||
{
|
||||
/// <summary>
|
||||
/// Does not store any state to durable storage, just keeps it in memory.
|
||||
/// Intended for testing scenarios.
|
||||
/// </summary>
|
||||
Memory = 0,
|
||||
|
||||
/// <summary>
|
||||
/// Uses the Faster key-value store.
|
||||
/// </summary>
|
||||
Faster = 1,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Configuration options for the transport component
|
||||
/// </summary>
|
||||
public enum TransportChoices
|
||||
{
|
||||
/// <summary>
|
||||
/// Passes messages through memory and puts all partitions on a single host
|
||||
/// Intended for testing scenarios.
|
||||
/// </summary>
|
||||
Memory = 0,
|
||||
|
||||
/// <summary>
|
||||
/// Passes messages through eventhubs; can distribute over multiple machines via
|
||||
/// the eventhubs EventProcessor.
|
||||
/// </summary>
|
||||
EventHubs = 1,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines the components to use given a transport connection string.
|
||||
/// </summary>
|
||||
public static void Parse(string transportConnectionString, out StorageChoices storage, out TransportChoices transport, out int? numPartitions)
|
||||
{
|
||||
if (transportConnectionString.StartsWith("Memory"))
|
||||
{
|
||||
transport = TransportChoices.Memory;
|
||||
storage = transportConnectionString.StartsWith("MemoryF") ? StorageChoices.Faster : StorageChoices.Memory;
|
||||
numPartitions = int.Parse(transportConnectionString.Substring(transportConnectionString.IndexOf(":") + 1));
|
||||
}
|
||||
else
|
||||
{
|
||||
transport = TransportChoices.EventHubs;
|
||||
storage = StorageChoices.Faster;
|
||||
numPartitions = null; // number of partitions is detected dynamically, not specified by transportConnectionString
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns the name of the eventhubs namespace
|
||||
/// </summary>
|
||||
public static string EventHubsNamespaceName(string transportConnectionString)
|
||||
{
|
||||
var builder = new EventHubsConnectionStringBuilder(transportConnectionString);
|
||||
var host = builder.Endpoint.Host;
|
||||
return host.Substring(0, host.IndexOf('.'));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,402 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Linq;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using DurableTask.Core;
|
||||
using DurableTask.Core.History;
|
||||
using DurableTask.Netherite.Scaling;
|
||||
using Dynamitey;
|
||||
|
||||
[DataContract]
|
||||
class ActivitiesState : TrackedObject
|
||||
{
|
||||
[DataMember]
|
||||
public Dictionary<long, TaskMessage> Pending { get; private set; }
|
||||
|
||||
[DataMember]
|
||||
public Queue<ActivityInfo> LocalBacklog { get; private set; }
|
||||
|
||||
[DataMember]
|
||||
public Queue<ActivityInfo> QueuedRemotes { get; private set; }
|
||||
|
||||
[DataMember]
|
||||
public int[] ReportedRemoteLoads { get; private set; }
|
||||
|
||||
[DataMember]
|
||||
public int EstimatedLocalWorkItemLoad { get; private set; }
|
||||
|
||||
[DataMember]
|
||||
public long SequenceNumber { get; set; }
|
||||
|
||||
[IgnoreDataMember]
|
||||
public override TrackedObjectKey Key => new TrackedObjectKey(TrackedObjectKey.TrackedObjectType.Activities);
|
||||
|
||||
public static string GetWorkItemId(uint partition, long activityId) => $"{partition:D2}-A{activityId}";
|
||||
|
||||
public override void OnFirstInitialization()
|
||||
{
|
||||
this.Pending = new Dictionary<long, TaskMessage>();
|
||||
this.LocalBacklog = new Queue<ActivityInfo>();
|
||||
this.QueuedRemotes = new Queue<ActivityInfo>();
|
||||
this.ReportedRemoteLoads = new int[this.Partition.NumberPartitions()];
|
||||
uint numberPartitions = this.Partition.NumberPartitions();
|
||||
for (uint i = 0; i < numberPartitions; i++)
|
||||
{
|
||||
this.ReportedRemoteLoads[i] = NOT_CONTACTED;
|
||||
}
|
||||
}
|
||||
|
||||
const int NOT_CONTACTED = -1;
|
||||
const int RESPONSE_PENDING = int.MaxValue;
|
||||
|
||||
const int ABSOLUTE_LOAD_LIMIT_FOR_REMOTES = 1000;
|
||||
const double RELATIVE_LOAD_LIMIT_FOR_REMOTES = .8;
|
||||
const double PORTION_SUBJECT_TO_OFFLOAD = .5;
|
||||
const int OFFLOAD_MAX_BATCH_SIZE = 100;
|
||||
const int OFFLOAD_MIN_BATCH_SIZE = 10;
|
||||
|
||||
const int MAX_WORKITEM_LOAD = 10;
|
||||
|
||||
// minimum time for an activity to be waiting before it is offloaded to a remote partition
|
||||
static readonly TimeSpan WaitTimeThresholdForOffload = TimeSpan.FromSeconds(15);
|
||||
|
||||
public override void OnRecoveryCompleted()
|
||||
{
|
||||
// reschedule work items
|
||||
foreach (var pending in this.Pending)
|
||||
{
|
||||
this.Partition.EnqueueActivityWorkItem(new ActivityWorkItem(this.Partition, pending.Key, pending.Value));
|
||||
}
|
||||
|
||||
if (this.LocalBacklog.Count > 0)
|
||||
{
|
||||
this.ScheduleNextOffloadDecision(WaitTimeThresholdForOffload);
|
||||
}
|
||||
}
|
||||
|
||||
public override void UpdateLoadInfo(PartitionLoadInfo info)
|
||||
{
|
||||
info.Activities = this.Pending.Count + this.LocalBacklog.Count + this.QueuedRemotes.Count;
|
||||
info.ActivityLatencyMs = Enumerable.Concat(this.LocalBacklog, this.QueuedRemotes)
|
||||
.Select(a => (long)(DateTime.UtcNow - a.IssueTime).TotalMilliseconds)
|
||||
.DefaultIfEmpty()
|
||||
.Max();
|
||||
}
|
||||
|
||||
public override string ToString()
|
||||
{
|
||||
return $"Activities ({this.Pending.Count} pending) next={this.SequenceNumber:D6}";
|
||||
}
|
||||
|
||||
void ScheduleNextOffloadDecision(TimeSpan delay)
|
||||
{
|
||||
this.Partition.PendingTimers.Schedule(DateTime.UtcNow + delay, new OffloadDecision()
|
||||
{
|
||||
PartitionId = this.Partition.PartitionId,
|
||||
Timestamp = DateTime.UtcNow + delay,
|
||||
});
|
||||
}
|
||||
|
||||
public bool TryGetNextActivity(out ActivityInfo activityInfo)
|
||||
{
|
||||
// take the most recent from the backlog or the queued remotes
|
||||
if (this.LocalBacklog.Count > 0 &&
|
||||
!(this.QueuedRemotes.Count > 0 && this.QueuedRemotes.Peek().IssueTime < this.LocalBacklog.Peek().IssueTime))
|
||||
{
|
||||
activityInfo = this.LocalBacklog.Dequeue();
|
||||
return true;
|
||||
}
|
||||
else if (this.QueuedRemotes.Count > 0)
|
||||
{
|
||||
activityInfo = this.QueuedRemotes.Dequeue();
|
||||
return true;
|
||||
}
|
||||
else
|
||||
{
|
||||
activityInfo = null;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public void Process(BatchProcessed evt, EffectTracker effects)
|
||||
{
|
||||
// the completed orchestration work item can launch activities
|
||||
foreach (var msg in evt.ActivityMessages)
|
||||
{
|
||||
var activityId = this.SequenceNumber++;
|
||||
|
||||
if (this.Pending.Count == 0 || this.EstimatedLocalWorkItemLoad <= MAX_WORKITEM_LOAD)
|
||||
{
|
||||
this.Pending.Add(activityId, msg);
|
||||
|
||||
if (!effects.IsReplaying)
|
||||
{
|
||||
this.Partition.EnqueueActivityWorkItem(new ActivityWorkItem(this.Partition, activityId, msg));
|
||||
}
|
||||
|
||||
this.EstimatedLocalWorkItemLoad++;
|
||||
}
|
||||
else
|
||||
{
|
||||
this.LocalBacklog.Enqueue(new ActivityInfo()
|
||||
{
|
||||
ActivityId = activityId,
|
||||
IssueTime = evt.Timestamp,
|
||||
Message = msg,
|
||||
});
|
||||
|
||||
if (!effects.IsReplaying && this.LocalBacklog.Count == 1)
|
||||
{
|
||||
this.ScheduleNextOffloadDecision(WaitTimeThresholdForOffload);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void Process(ActivityOffloadReceived evt, EffectTracker effects)
|
||||
{
|
||||
// may bring in offloaded activities from other partitions
|
||||
foreach (var msg in evt.OffloadedActivities)
|
||||
{
|
||||
var activityId = this.SequenceNumber++;
|
||||
|
||||
if (this.Pending.Count == 0 || this.EstimatedLocalWorkItemLoad <= MAX_WORKITEM_LOAD)
|
||||
{
|
||||
this.Pending.Add(activityId, msg);
|
||||
|
||||
if (!effects.IsReplaying)
|
||||
{
|
||||
this.Partition.EnqueueActivityWorkItem(new ActivityWorkItem(this.Partition, activityId, msg));
|
||||
}
|
||||
|
||||
this.EstimatedLocalWorkItemLoad++;
|
||||
}
|
||||
else
|
||||
{
|
||||
this.QueuedRemotes.Enqueue(new ActivityInfo()
|
||||
{
|
||||
ActivityId = activityId,
|
||||
IssueTime = evt.Timestamp,
|
||||
Message = msg,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void Process(ActivityCompleted evt, EffectTracker effects)
|
||||
{
|
||||
// records the result of a finished activity and launches an offload decision
|
||||
|
||||
this.Pending.Remove(evt.ActivityId);
|
||||
|
||||
if (evt.OriginPartitionId == effects.Partition.PartitionId)
|
||||
{
|
||||
this.EstimatedLocalWorkItemLoad = evt.ReportedLoad;
|
||||
// the response can be delivered to a session on this partition
|
||||
effects.Add(TrackedObjectKey.Sessions);
|
||||
}
|
||||
else
|
||||
{
|
||||
// the response must be sent to a remote partition
|
||||
evt.ReportedLoad = this.LocalBacklog.Count + this.QueuedRemotes.Count;
|
||||
effects.Add(TrackedObjectKey.Outbox);
|
||||
}
|
||||
|
||||
// now that an activity has completed, we can perhaps add more from the backlog
|
||||
while (this.Pending.Count == 0 || this.EstimatedLocalWorkItemLoad <= MAX_WORKITEM_LOAD)
|
||||
{
|
||||
if (this.TryGetNextActivity(out var activityInfo))
|
||||
{
|
||||
this.Pending.Add(activityInfo.ActivityId, activityInfo.Message);
|
||||
|
||||
if (!effects.IsReplaying)
|
||||
{
|
||||
this.Partition.EnqueueActivityWorkItem(new ActivityWorkItem(this.Partition, activityInfo.ActivityId, activityInfo.Message));
|
||||
}
|
||||
|
||||
this.EstimatedLocalWorkItemLoad++;
|
||||
}
|
||||
else
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void Process(RemoteActivityResultReceived evt, EffectTracker effects)
|
||||
{
|
||||
// records the reported queue size
|
||||
this.ReportedRemoteLoads[evt.OriginPartition] = evt.ActivitiesQueueSize;
|
||||
}
|
||||
|
||||
public void Process(OffloadDecision offloadDecisionEvent, EffectTracker effects)
|
||||
{
|
||||
// check for offload conditions and if satisfied, send batch to remote
|
||||
|
||||
if (this.LocalBacklog.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
// find how many offload candidates we have
|
||||
int numberOffloadCandidates = this.CountOffloadCandidates(offloadDecisionEvent.Timestamp);
|
||||
|
||||
if (numberOffloadCandidates < OFFLOAD_MIN_BATCH_SIZE)
|
||||
{
|
||||
return; // no offloading if we cannot offload enough
|
||||
}
|
||||
|
||||
if (this.FindOffloadTarget(numberOffloadCandidates, out uint target, out int maxBatchsize))
|
||||
{
|
||||
// don't pick this same target again until we get a response telling us the current queue size
|
||||
var targetLoad = this.ReportedRemoteLoads[target];
|
||||
this.ReportedRemoteLoads[target] = RESPONSE_PENDING;
|
||||
|
||||
// we are adding (nonpersisted) information to the event just as a way of passing it to the OutboxState
|
||||
offloadDecisionEvent.DestinationPartitionId = target;
|
||||
offloadDecisionEvent.OffloadedActivities = new List<TaskMessage>();
|
||||
|
||||
for (int i = 0; i < maxBatchsize; i++)
|
||||
{
|
||||
var info = this.LocalBacklog.Dequeue();
|
||||
offloadDecisionEvent.OffloadedActivities.Add(info.Message);
|
||||
|
||||
if (this.LocalBacklog.Count == 0 || offloadDecisionEvent.Timestamp - this.LocalBacklog.Peek().IssueTime < WaitTimeThresholdForOffload)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// process this on OutboxState so the events get sent
|
||||
effects.Add(TrackedObjectKey.Outbox);
|
||||
|
||||
var reportedRemotes = string.Join(",", this.ReportedRemoteLoads.Select((int x, int i) =>
|
||||
(i == target) ? $"{targetLoad}+{offloadDecisionEvent.OffloadedActivities.Count}" : (x == NOT_CONTACTED ? "-" : (x == RESPONSE_PENDING ? "X" : x.ToString()))));
|
||||
|
||||
this.Partition.EventTraceHelper.TracePartitionOffloadDecision(this.EstimatedLocalWorkItemLoad, this.Pending.Count, this.LocalBacklog.Count, this.QueuedRemotes.Count, reportedRemotes);
|
||||
|
||||
// try again relatively soon
|
||||
this.ScheduleNextOffloadDecision(TimeSpan.FromMilliseconds(200));
|
||||
}
|
||||
else
|
||||
{
|
||||
var reportedRemotes = string.Join(",", this.ReportedRemoteLoads.Select((int x) => x == NOT_CONTACTED ? "-" : (x == RESPONSE_PENDING ? "X" : x.ToString())));
|
||||
|
||||
this.Partition.EventTraceHelper.TracePartitionOffloadDecision(this.EstimatedLocalWorkItemLoad, this.Pending.Count, this.LocalBacklog.Count, this.QueuedRemotes.Count, reportedRemotes);
|
||||
|
||||
// there are no eligible recipients... try again in a while
|
||||
this.ScheduleNextOffloadDecision(TimeSpan.FromSeconds(10));
|
||||
}
|
||||
}
|
||||
|
||||
int CountOffloadCandidates(DateTime now)
|
||||
{
|
||||
int numberOffloadCandidates = 0;
|
||||
int limit = (int)(Math.Min(PORTION_SUBJECT_TO_OFFLOAD * this.LocalBacklog.Count, OFFLOAD_MAX_BATCH_SIZE * this.Partition.NumberPartitions()));
|
||||
foreach (var entry in this.LocalBacklog)
|
||||
{
|
||||
if (now - entry.IssueTime < WaitTimeThresholdForOffload
|
||||
|| numberOffloadCandidates++ > limit)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
return numberOffloadCandidates;
|
||||
}
|
||||
|
||||
bool FindOffloadTarget(double portionSubjectToOffload, out uint target, out int batchsize)
|
||||
{
|
||||
uint numberPartitions = this.Partition.NumberPartitions();
|
||||
uint? firstNotContacted = null;
|
||||
uint? eligibleTargetWithSmallestQueue = null;
|
||||
int minimalReportedQueueSizeFound = int.MaxValue;
|
||||
double estimatedParallism = 0;
|
||||
double remoteLoadLimit = Math.Min(ABSOLUTE_LOAD_LIMIT_FOR_REMOTES, RELATIVE_LOAD_LIMIT_FOR_REMOTES * this.LocalBacklog.Count);
|
||||
|
||||
for (uint i = 0; i < numberPartitions - 1; i++)
|
||||
{
|
||||
uint candidate = (this.Partition.PartitionId + i + 1) % numberPartitions;
|
||||
int reported = this.ReportedRemoteLoads[candidate];
|
||||
if (reported == NOT_CONTACTED)
|
||||
{
|
||||
if (!firstNotContacted.HasValue)
|
||||
{
|
||||
firstNotContacted = candidate;
|
||||
}
|
||||
estimatedParallism += 1;
|
||||
}
|
||||
else if (reported != RESPONSE_PENDING)
|
||||
{
|
||||
if (reported < remoteLoadLimit)
|
||||
{
|
||||
estimatedParallism += 1;
|
||||
if (reported < minimalReportedQueueSizeFound)
|
||||
{
|
||||
minimalReportedQueueSizeFound = reported;
|
||||
eligibleTargetWithSmallestQueue = candidate;
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
estimatedParallism += 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (eligibleTargetWithSmallestQueue.HasValue)
|
||||
{
|
||||
// we found a lowly loaded target
|
||||
target = eligibleTargetWithSmallestQueue.Value;
|
||||
batchsize = Math.Min(OFFLOAD_MAX_BATCH_SIZE, Math.Max(OFFLOAD_MIN_BATCH_SIZE, (int)Math.Ceiling(portionSubjectToOffload / estimatedParallism)));
|
||||
if (minimalReportedQueueSizeFound < batchsize)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
if (firstNotContacted.HasValue)
|
||||
{
|
||||
// we did not find a lowly loaded target with enough spare capacity
|
||||
target = firstNotContacted.Value;
|
||||
batchsize = OFFLOAD_MIN_BATCH_SIZE;
|
||||
return true;
|
||||
}
|
||||
|
||||
target = 0;
|
||||
batchsize = 0;
|
||||
return false;
|
||||
}
|
||||
|
||||
[DataContract]
|
||||
public class ActivityInfo
|
||||
{
|
||||
[DataMember]
|
||||
public long ActivityId;
|
||||
|
||||
[DataMember]
|
||||
public TaskMessage Message;
|
||||
|
||||
[DataMember]
|
||||
public DateTime IssueTime;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,87 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using DurableTask.Core;
|
||||
using DurableTask.Core.History;
|
||||
using DurableTask.Netherite.Faster;
|
||||
|
||||
[DataContract]
|
||||
class DedupState : TrackedObject
|
||||
{
|
||||
[DataMember]
|
||||
public Dictionary<uint, long> LastProcessed { get; set; } = new Dictionary<uint, long>();
|
||||
|
||||
[DataMember]
|
||||
public (long, long) Positions; // used by FasterAlt to persist positions
|
||||
|
||||
[IgnoreDataMember]
|
||||
public override TrackedObjectKey Key => new TrackedObjectKey(TrackedObjectKey.TrackedObjectType.Dedup);
|
||||
|
||||
bool IsNotDuplicate(PartitionMessageEvent evt)
|
||||
{
|
||||
// detect duplicates of incoming partition-to-partition events by comparing commit log position of this event against last processed event from same partition
|
||||
this.LastProcessed.TryGetValue(evt.OriginPartition, out long lastProcessed);
|
||||
if (evt.OriginPosition > lastProcessed)
|
||||
{
|
||||
this.LastProcessed[evt.OriginPartition] = evt.OriginPosition;
|
||||
return true;
|
||||
}
|
||||
else
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public void Process(ActivityOffloadReceived evt, EffectTracker effects)
|
||||
{
|
||||
// queues activities originating from a remote partition to execute on this partition
|
||||
if (this.IsNotDuplicate(evt))
|
||||
{
|
||||
effects.Add(TrackedObjectKey.Activities);
|
||||
}
|
||||
}
|
||||
|
||||
public void Process(RemoteActivityResultReceived evt, EffectTracker effects)
|
||||
{
|
||||
// returns a response to an ongoing orchestration, and reports load data to the offload logic
|
||||
if (this.IsNotDuplicate(evt))
|
||||
{
|
||||
effects.Add(TrackedObjectKey.Sessions);
|
||||
effects.Add(TrackedObjectKey.Activities);
|
||||
}
|
||||
}
|
||||
|
||||
public void Process(TaskMessagesReceived evt, EffectTracker effects)
|
||||
{
|
||||
// contains messages to be processed by sessions and/or to be scheduled by timer
|
||||
if (this.IsNotDuplicate(evt))
|
||||
{
|
||||
if (evt.TaskMessages != null)
|
||||
{
|
||||
effects.Add(TrackedObjectKey.Sessions);
|
||||
}
|
||||
if (evt.DelayedTaskMessages != null)
|
||||
{
|
||||
effects.Add(TrackedObjectKey.Timers);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,122 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using DurableTask.Core;
|
||||
using DurableTask.Core.Exceptions;
|
||||
using DurableTask.Core.History;
|
||||
|
||||
[DataContract]
|
||||
class HistoryState : TrackedObject
|
||||
{
|
||||
[DataMember]
|
||||
public string InstanceId { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public string ExecutionId { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public List<HistoryEvent> History { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public int Episode { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// We cache this so we can resume the execution at the execution cursor.
|
||||
/// </summary>
|
||||
[IgnoreDataMember]
|
||||
public OrchestrationWorkItem CachedOrchestrationWorkItem { get; set; }
|
||||
|
||||
[IgnoreDataMember]
|
||||
public override TrackedObjectKey Key => new TrackedObjectKey(TrackedObjectKey.TrackedObjectType.History, this.InstanceId);
|
||||
|
||||
public override string ToString()
|
||||
{
|
||||
return $"History InstanceId={this.InstanceId} ExecutionId={this.ExecutionId} Events={this.History.Count}";
|
||||
}
|
||||
|
||||
void DeleteHistory()
|
||||
{
|
||||
this.History = null;
|
||||
this.Episode = 0;
|
||||
this.ExecutionId = null;
|
||||
this.CachedOrchestrationWorkItem = null;
|
||||
}
|
||||
|
||||
public void Process(BatchProcessed evt, EffectTracker effects)
|
||||
{
|
||||
// can add events to the history, or replace it with a new history
|
||||
|
||||
// update the stored history
|
||||
if (this.History == null || evt.State.OrchestrationInstance.ExecutionId != this.ExecutionId)
|
||||
{
|
||||
this.History = new List<HistoryEvent>();
|
||||
this.Episode = 0;
|
||||
this.ExecutionId = evt.State.OrchestrationInstance.ExecutionId;
|
||||
}
|
||||
|
||||
this.Partition.Assert(!string.IsNullOrEmpty(this.InstanceId) || string.IsNullOrEmpty(this.ExecutionId));
|
||||
|
||||
// add all the new events to the history, and update episode number
|
||||
if (evt.NewEvents != null)
|
||||
{
|
||||
for (int i = 0; i < evt.NewEvents.Count; i++)
|
||||
{
|
||||
var historyEvent = evt.NewEvents[i];
|
||||
if (historyEvent.EventType == EventType.OrchestratorStarted)
|
||||
{
|
||||
this.Episode++;
|
||||
}
|
||||
this.History.Add(evt.NewEvents[i]);
|
||||
}
|
||||
}
|
||||
|
||||
if (!effects.IsReplaying)
|
||||
{
|
||||
this.Partition.EventTraceHelper.TraceInstanceUpdate(
|
||||
evt.WorkItemId,
|
||||
evt.State.OrchestrationInstance.InstanceId,
|
||||
evt.State.OrchestrationInstance.ExecutionId,
|
||||
this.History.Count,
|
||||
evt.NewEvents, this.Episode);
|
||||
|
||||
// if present, we keep the work item so we can reuse the execution cursor
|
||||
this.CachedOrchestrationWorkItem = evt.WorkItemForReuse;
|
||||
|
||||
if (this.CachedOrchestrationWorkItem != null && this.CachedOrchestrationWorkItem.OrchestrationRuntimeState?.OrchestrationInstance?.ExecutionId != evt.State.OrchestrationInstance.ExecutionId)
|
||||
{
|
||||
effects.Partition.EventTraceHelper.TraceWarning($"Dropping bad workitem cache instance={this.InstanceId} expected_executionid={evt.State.OrchestrationInstance.ExecutionId} actual_executionid={this.CachedOrchestrationWorkItem.OrchestrationRuntimeState?.OrchestrationInstance?.ExecutionId}");
|
||||
this.CachedOrchestrationWorkItem = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void Process(DeletionRequestReceived deletionRequestReceived, EffectTracker effects)
|
||||
{
|
||||
this.DeleteHistory();
|
||||
}
|
||||
|
||||
public void Process(PurgeBatchIssued purgeBatchIssued, EffectTracker effects)
|
||||
{
|
||||
this.DeleteHistory();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,182 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Text;
|
||||
using DurableTask.Core;
|
||||
using DurableTask.Core.Exceptions;
|
||||
using DurableTask.Core.History;
|
||||
|
||||
[DataContract]
|
||||
class InstanceState : TrackedObject
|
||||
{
|
||||
[DataMember]
|
||||
public string InstanceId { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public OrchestrationState OrchestrationState { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public List<WaitRequestReceived> Waiters { get; set; }
|
||||
|
||||
[IgnoreDataMember]
|
||||
public override TrackedObjectKey Key => new TrackedObjectKey(TrackedObjectKey.TrackedObjectType.Instance, this.InstanceId);
|
||||
|
||||
public override string ToString()
|
||||
{
|
||||
return $"History InstanceId={this.InstanceId} Status={this.OrchestrationState?.OrchestrationStatus}";
|
||||
}
|
||||
|
||||
public void Process(CreationRequestReceived creationRequestReceived, EffectTracker effects)
|
||||
{
|
||||
bool filterDuplicate = this.OrchestrationState != null
|
||||
&& creationRequestReceived.DedupeStatuses != null
|
||||
&& creationRequestReceived.DedupeStatuses.Contains(this.OrchestrationState.OrchestrationStatus);
|
||||
|
||||
if (!filterDuplicate)
|
||||
{
|
||||
var ee = creationRequestReceived.ExecutionStartedEvent;
|
||||
|
||||
// set the orchestration state now (before processing the creation in the history)
|
||||
// so that this new instance is "on record" immediately - it is guaranteed to replace whatever is in flight
|
||||
this.OrchestrationState = new OrchestrationState
|
||||
{
|
||||
Name = ee.Name,
|
||||
Version = ee.Version,
|
||||
OrchestrationInstance = ee.OrchestrationInstance,
|
||||
OrchestrationStatus = OrchestrationStatus.Pending,
|
||||
ParentInstance = ee.ParentInstance,
|
||||
Input = ee.Input,
|
||||
Tags = ee.Tags,
|
||||
CreatedTime = ee.Timestamp,
|
||||
LastUpdatedTime = DateTime.UtcNow,
|
||||
CompletedTime = Core.Common.DateTimeUtils.MinDateTime,
|
||||
ScheduledStartTime = ee.ScheduledStartTime
|
||||
};
|
||||
|
||||
// queue the message in the session, or start a timer if delayed
|
||||
if (!ee.ScheduledStartTime.HasValue)
|
||||
{
|
||||
effects.Add(TrackedObjectKey.Sessions);
|
||||
}
|
||||
else
|
||||
{
|
||||
effects.Add(TrackedObjectKey.Timers);
|
||||
}
|
||||
}
|
||||
|
||||
if (!effects.IsReplaying)
|
||||
{
|
||||
// send response to client
|
||||
effects.Partition.Send(new CreationResponseReceived()
|
||||
{
|
||||
ClientId = creationRequestReceived.ClientId,
|
||||
RequestId = creationRequestReceived.RequestId,
|
||||
Succeeded = !filterDuplicate,
|
||||
ExistingInstanceOrchestrationStatus = this.OrchestrationState?.OrchestrationStatus,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void Process(BatchProcessed evt, EffectTracker effects)
|
||||
{
|
||||
// update the state of an orchestration
|
||||
this.OrchestrationState = evt.State;
|
||||
|
||||
// if the orchestration is complete, notify clients that are waiting for it
|
||||
if (this.Waiters != null && WaitRequestReceived.SatisfiesWaitCondition(this.OrchestrationState))
|
||||
{
|
||||
if (!effects.IsReplaying)
|
||||
{
|
||||
foreach (var request in this.Waiters)
|
||||
{
|
||||
this.Partition.Send(request.CreateResponse(this.OrchestrationState));
|
||||
}
|
||||
}
|
||||
|
||||
this.Waiters = null;
|
||||
}
|
||||
}
|
||||
|
||||
public void Process(WaitRequestReceived evt, EffectTracker effects)
|
||||
{
|
||||
if (WaitRequestReceived.SatisfiesWaitCondition(this.OrchestrationState))
|
||||
{
|
||||
if (!effects.IsReplaying)
|
||||
{
|
||||
this.Partition.Send(evt.CreateResponse(this.OrchestrationState));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (this.Waiters == null)
|
||||
{
|
||||
this.Waiters = new List<WaitRequestReceived>();
|
||||
}
|
||||
else
|
||||
{
|
||||
// cull the list of waiters to remove requests that have already timed out
|
||||
this.Waiters = this.Waiters
|
||||
.Where(request => request.TimeoutUtc > DateTime.UtcNow)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
this.Waiters.Add(evt);
|
||||
}
|
||||
}
|
||||
|
||||
public void Process(DeletionRequestReceived deletionRequest, EffectTracker effects)
|
||||
{
|
||||
int numberInstancesDeleted = 0;
|
||||
|
||||
if (this.OrchestrationState != null
|
||||
&& (!deletionRequest.CreatedTime.HasValue || deletionRequest.CreatedTime.Value == this.OrchestrationState.CreatedTime))
|
||||
{
|
||||
this.OrchestrationState = null;
|
||||
numberInstancesDeleted++;
|
||||
|
||||
// we also delete this instance's history, and pending operations on it
|
||||
effects.Add(TrackedObjectKey.History(this.InstanceId));
|
||||
effects.Add(TrackedObjectKey.Sessions);
|
||||
}
|
||||
|
||||
if (!effects.IsReplaying)
|
||||
{
|
||||
this.Partition.Send(new DeletionResponseReceived()
|
||||
{
|
||||
ClientId = deletionRequest.ClientId,
|
||||
RequestId = deletionRequest.RequestId,
|
||||
NumberInstancesDeleted = numberInstancesDeleted,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
public void Process(PurgeBatchIssued purgeBatchIssued, EffectTracker effects)
|
||||
{
|
||||
OrchestrationState state = this.OrchestrationState;
|
||||
if (this.OrchestrationState != null
|
||||
&& purgeBatchIssued.InstanceQuery.Matches(this.OrchestrationState))
|
||||
{
|
||||
this.OrchestrationState = null;
|
||||
purgeBatchIssued.Purged.Add(this.InstanceId);
|
||||
effects.Add(TrackedObjectKey.History(this.InstanceId));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,214 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using DurableTask.Core;
|
||||
using DurableTask.Core.Common;
|
||||
using DurableTask.Core.History;
|
||||
using DurableTask.Netherite.Scaling;
|
||||
|
||||
[DataContract]
|
||||
class OutboxState : TrackedObject, TransportAbstraction.IDurabilityListener
|
||||
{
|
||||
[DataMember]
|
||||
public SortedDictionary<long, Batch> Outbox { get; private set; } = new SortedDictionary<long, Batch>();
|
||||
|
||||
public override TrackedObjectKey Key => new TrackedObjectKey(TrackedObjectKey.TrackedObjectType.Outbox);
|
||||
|
||||
public override void OnRecoveryCompleted()
|
||||
{
|
||||
// resend all pending
|
||||
foreach (var kvp in this.Outbox)
|
||||
{
|
||||
// recover non-persisted fields
|
||||
kvp.Value.Position = kvp.Key;
|
||||
kvp.Value.Partition = this.Partition;
|
||||
|
||||
// resend (anything we have recovered is of course persisted)
|
||||
this.Partition.EventDetailTracer?.TraceEventProcessingDetail($"Resent {kvp.Key:D10} ({kvp.Value} messages)");
|
||||
this.Send(kvp.Value);
|
||||
}
|
||||
}
|
||||
|
||||
public override void UpdateLoadInfo(PartitionLoadInfo info)
|
||||
{
|
||||
info.Outbox = this.Outbox.Count;
|
||||
}
|
||||
|
||||
public override string ToString()
|
||||
{
|
||||
return $"Outbox ({this.Outbox.Count} pending)";
|
||||
}
|
||||
|
||||
void SendBatchOnceEventIsPersisted(PartitionUpdateEvent evt, EffectTracker effects, Batch batch)
|
||||
{
|
||||
// put the messages in the outbox where they are kept until actually sent
|
||||
var commitPosition = evt.NextCommitLogPosition;
|
||||
|
||||
// Update the ready to send timestamp to check the delay caused
|
||||
// by non-speculation
|
||||
evt.ReadyToSendTimestamp = this.Partition.CurrentTimeMs;
|
||||
|
||||
this.Outbox[commitPosition] = batch;
|
||||
batch.Position = commitPosition;
|
||||
batch.Partition = this.Partition;
|
||||
|
||||
if (!effects.IsReplaying)
|
||||
{
|
||||
if (!this.Partition.Settings.PersistStepsFirst)
|
||||
{
|
||||
// we must not send messages until this step has been persisted
|
||||
evt.OutboxBatch = batch;
|
||||
DurabilityListeners.Register(evt, this);
|
||||
}
|
||||
else
|
||||
{
|
||||
// we can send the messages now
|
||||
this.Send(batch);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void ConfirmDurable(Event evt)
|
||||
{
|
||||
var partitionUpdateEvent = ((PartitionUpdateEvent)evt);
|
||||
|
||||
// Calculate the delay by not sending immediately
|
||||
partitionUpdateEvent.SentTimestamp = this.Partition.CurrentTimeMs;
|
||||
this.Partition.EventTraceHelper.TraceEventSentDetail(partitionUpdateEvent);
|
||||
|
||||
long commitPosition = partitionUpdateEvent.NextCommitLogPosition;
|
||||
this.Partition.EventDetailTracer?.TraceEventProcessingDetail($"Store has persisted event {evt} id={evt.EventIdString}, now sending messages");
|
||||
this.Send(partitionUpdateEvent.OutboxBatch);
|
||||
}
|
||||
|
||||
void Send(Batch batch)
|
||||
{
|
||||
// now that we know the sending event is persisted, we can send the messages
|
||||
foreach (var outmessage in batch.OutgoingMessages)
|
||||
{
|
||||
DurabilityListeners.Register(outmessage, batch);
|
||||
outmessage.OriginPartition = this.Partition.PartitionId;
|
||||
outmessage.OriginPosition = batch.Position;
|
||||
//outmessage.SentTimestampUnixMs = DateTimeOffset.Now.ToUnixTimeMilliseconds();
|
||||
this.Partition.Send(outmessage);
|
||||
}
|
||||
}
|
||||
|
||||
[DataContract]
|
||||
public class Batch : TransportAbstraction.IDurabilityListener
|
||||
{
|
||||
[DataMember]
|
||||
public List<PartitionMessageEvent> OutgoingMessages { get; set; } = new List<PartitionMessageEvent>();
|
||||
|
||||
[IgnoreDataMember]
|
||||
public long Position { get; set; }
|
||||
|
||||
[IgnoreDataMember]
|
||||
public Partition Partition { get; set; }
|
||||
|
||||
[IgnoreDataMember]
|
||||
int numAcks = 0;
|
||||
|
||||
public void ConfirmDurable(Event evt)
|
||||
{
|
||||
this.Partition.EventDetailTracer?.TraceEventProcessingDetail($"Transport has confirmed event {evt} id={evt.EventIdString}");
|
||||
|
||||
if (++this.numAcks == this.OutgoingMessages.Count)
|
||||
{
|
||||
this.Partition.SubmitInternalEvent(new SendConfirmed()
|
||||
{
|
||||
PartitionId = this.Partition.PartitionId,
|
||||
Position = Position,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void Process(SendConfirmed evt, EffectTracker _)
|
||||
{
|
||||
this.Partition.EventDetailTracer?.TraceEventProcessingDetail($"Store has sent all outbound messages by event {evt} id={evt.EventIdString}");
|
||||
|
||||
// we no longer need to keep these events around
|
||||
this.Outbox.Remove(evt.Position);
|
||||
}
|
||||
|
||||
public void Process(ActivityCompleted evt, EffectTracker effects)
|
||||
{
|
||||
var batch = new Batch();
|
||||
batch.OutgoingMessages.Add(new RemoteActivityResultReceived()
|
||||
{
|
||||
PartitionId = evt.OriginPartitionId,
|
||||
Result = evt.Response,
|
||||
ActivityId = evt.ActivityId,
|
||||
ActivitiesQueueSize = evt.ReportedLoad,
|
||||
});
|
||||
this.SendBatchOnceEventIsPersisted(evt, effects, batch);
|
||||
}
|
||||
|
||||
public void Process(BatchProcessed evt, EffectTracker effects)
|
||||
{
|
||||
var sorted = new Dictionary<uint, TaskMessagesReceived>();
|
||||
foreach (var message in evt.RemoteMessages)
|
||||
{
|
||||
var instanceId = message.OrchestrationInstance.InstanceId;
|
||||
var destination = this.Partition.PartitionFunction(instanceId);
|
||||
if (!sorted.TryGetValue(destination, out var outmessage))
|
||||
{
|
||||
sorted[destination] = outmessage = new TaskMessagesReceived()
|
||||
{
|
||||
PartitionId = destination,
|
||||
WorkItemId = evt.WorkItemId,
|
||||
};
|
||||
}
|
||||
if (Entities.IsDelayedEntityMessage(message, out _))
|
||||
{
|
||||
(outmessage.DelayedTaskMessages ?? (outmessage.DelayedTaskMessages = new List<TaskMessage>())).Add(message);
|
||||
}
|
||||
else if (message.Event is ExecutionStartedEvent executionStartedEvent && executionStartedEvent.ScheduledStartTime.HasValue)
|
||||
{
|
||||
(outmessage.DelayedTaskMessages ?? (outmessage.DelayedTaskMessages = new List<TaskMessage>())).Add(message);
|
||||
}
|
||||
else
|
||||
{
|
||||
(outmessage.TaskMessages ?? (outmessage.TaskMessages = new List<TaskMessage>())).Add(message);
|
||||
}
|
||||
}
|
||||
var batch = new Batch();
|
||||
batch.OutgoingMessages.AddRange(sorted.Values);
|
||||
this.SendBatchOnceEventIsPersisted(evt, effects, batch);
|
||||
}
|
||||
|
||||
public void Process(OffloadDecision evt, EffectTracker effects)
|
||||
{
|
||||
var batch = new Batch();
|
||||
batch.OutgoingMessages.Add(new ActivityOffloadReceived()
|
||||
{
|
||||
PartitionId = evt.DestinationPartitionId,
|
||||
OffloadedActivities = evt.OffloadedActivities,
|
||||
Timestamp = evt.Timestamp,
|
||||
});
|
||||
this.SendBatchOnceEventIsPersisted(evt, effects, batch);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,120 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using DurableTask.Core;
|
||||
using DurableTask.Core.History;
|
||||
using DurableTask.Netherite.Scaling;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
[DataContract]
|
||||
class PrefetchState : TrackedObject
|
||||
{
|
||||
[DataMember]
|
||||
public Dictionary<string, ClientRequestEventWithPrefetch> PendingPrefetches { get; private set; } = new Dictionary<string, ClientRequestEventWithPrefetch>();
|
||||
|
||||
[IgnoreDataMember]
|
||||
public override TrackedObjectKey Key => new TrackedObjectKey(TrackedObjectKey.TrackedObjectType.Prefetch);
|
||||
|
||||
public override void OnRecoveryCompleted()
|
||||
{
|
||||
// reissue prefetch tasks for what did not complete prior to crash/recovery
|
||||
foreach (var kvp in this.PendingPrefetches)
|
||||
{
|
||||
this.Partition.SubmitInternalEvent(new InstanceLookup(kvp.Value));
|
||||
}
|
||||
}
|
||||
|
||||
public override void UpdateLoadInfo(PartitionLoadInfo info)
|
||||
{
|
||||
info.Requests += this.PendingPrefetches.Count;
|
||||
}
|
||||
|
||||
public override string ToString()
|
||||
{
|
||||
return $"Prefetch ({this.PendingPrefetches.Count} pending)";
|
||||
}
|
||||
|
||||
public void Process(ClientRequestEventWithPrefetch clientRequestEvent, EffectTracker effects)
|
||||
{
|
||||
if (clientRequestEvent.Phase == ClientRequestEventWithPrefetch.ProcessingPhase.Read)
|
||||
{
|
||||
this.Partition.Assert(!this.PendingPrefetches.ContainsKey(clientRequestEvent.EventIdString));
|
||||
|
||||
// Issue a read request that fetches the instance state.
|
||||
// We have to buffer this request in the pending list so we can recover it.
|
||||
|
||||
this.PendingPrefetches.Add(clientRequestEvent.EventIdString, clientRequestEvent);
|
||||
|
||||
if (!effects.IsReplaying)
|
||||
{
|
||||
this.Partition.SubmitInternalEvent(new InstanceLookup(clientRequestEvent));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (this.PendingPrefetches.Remove(clientRequestEvent.EventIdString))
|
||||
{
|
||||
if (clientRequestEvent.Phase == ClientRequestEventWithPrefetch.ProcessingPhase.ConfirmAndProcess)
|
||||
{
|
||||
effects.Add(clientRequestEvent.Target);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
internal class InstanceLookup : InternalReadEvent
|
||||
{
|
||||
readonly ClientRequestEventWithPrefetch request;
|
||||
|
||||
public InstanceLookup(ClientRequestEventWithPrefetch clientRequest)
|
||||
{
|
||||
this.request = clientRequest;
|
||||
}
|
||||
|
||||
protected override void ExtraTraceInformation(StringBuilder s)
|
||||
{
|
||||
s.Append(':');
|
||||
s.Append(this.request.ToString());
|
||||
}
|
||||
|
||||
public override TrackedObjectKey ReadTarget => this.request.Target;
|
||||
|
||||
public override TrackedObjectKey? Prefetch => this.request.Prefetch;
|
||||
|
||||
public override EventId EventId => this.request.EventId;
|
||||
|
||||
public override void OnReadComplete(TrackedObject target, Partition partition)
|
||||
{
|
||||
partition.Assert(this.request.Phase == ClientRequestEventWithPrefetch.ProcessingPhase.Read);
|
||||
|
||||
bool requiresProcessing = this.request.OnReadComplete(target, partition);
|
||||
|
||||
var again = (ClientRequestEventWithPrefetch) this.request.Clone();
|
||||
|
||||
again.NextInputQueuePosition = 0; // this event is no longer considered an external event
|
||||
|
||||
again.Phase = requiresProcessing ?
|
||||
ClientRequestEventWithPrefetch.ProcessingPhase.ConfirmAndProcess : ClientRequestEventWithPrefetch.ProcessingPhase.Confirm;
|
||||
|
||||
partition.SubmitInternalEvent(again);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,123 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using DurableTask.Core;
|
||||
using DurableTask.Core.History;
|
||||
using DurableTask.Netherite.Scaling;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
[DataContract]
|
||||
class QueriesState : TrackedObject
|
||||
{
|
||||
[DataMember]
|
||||
public Dictionary<string, ClientRequestEventWithQuery> PendingQueries { get; private set; } = new Dictionary<string, ClientRequestEventWithQuery>();
|
||||
|
||||
[IgnoreDataMember]
|
||||
public override TrackedObjectKey Key => new TrackedObjectKey(TrackedObjectKey.TrackedObjectType.Queries);
|
||||
|
||||
public override void OnRecoveryCompleted()
|
||||
{
|
||||
// reissue queries that did not complete prior to crash/recovery
|
||||
foreach (var kvp in this.PendingQueries)
|
||||
{
|
||||
this.Partition.SubmitInternalEvent(new InstanceQueryEvent(kvp.Value));
|
||||
}
|
||||
}
|
||||
|
||||
public override void UpdateLoadInfo(PartitionLoadInfo info)
|
||||
{
|
||||
info.Requests += this.PendingQueries.Count;
|
||||
}
|
||||
|
||||
public override string ToString()
|
||||
{
|
||||
return $"Queries ({this.PendingQueries.Count} pending)";
|
||||
}
|
||||
|
||||
public void Process(ClientRequestEventWithQuery clientRequestEvent, EffectTracker effects)
|
||||
{
|
||||
if (clientRequestEvent.Phase == ClientRequestEventWithQuery.ProcessingPhase.Query)
|
||||
{
|
||||
this.Partition.Assert(!this.PendingQueries.ContainsKey(clientRequestEvent.EventIdString));
|
||||
|
||||
// Issue a read request that fetches the instance state.
|
||||
// We have to buffer this request in the pending list so we can recover it.
|
||||
|
||||
this.PendingQueries.Add(clientRequestEvent.EventIdString, clientRequestEvent);
|
||||
|
||||
if (!effects.IsReplaying)
|
||||
{
|
||||
this.Partition.SubmitInternalEvent(new InstanceQueryEvent(clientRequestEvent));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
this.PendingQueries.Remove(clientRequestEvent.EventIdString);
|
||||
}
|
||||
}
|
||||
|
||||
public void Process(PurgeBatchIssued purgeBatchIssued, EffectTracker effects)
|
||||
{
|
||||
var purgeRequest = (PurgeRequestReceived)this.PendingQueries[purgeBatchIssued.QueryEventId];
|
||||
purgeRequest.NumberInstancesPurged += purgeBatchIssued.Purged.Count;
|
||||
|
||||
if (!effects.IsReplaying)
|
||||
{
|
||||
// lets the query that is currently in progress know that this batch is done
|
||||
purgeBatchIssued.WhenProcessed.TrySetResult(null);
|
||||
}
|
||||
}
|
||||
|
||||
internal class InstanceQueryEvent : PartitionQueryEvent
|
||||
{
|
||||
readonly ClientRequestEventWithQuery request;
|
||||
|
||||
public InstanceQueryEvent(ClientRequestEventWithQuery clientRequest)
|
||||
{
|
||||
this.request = clientRequest;
|
||||
}
|
||||
|
||||
protected override void ExtraTraceInformation(StringBuilder s)
|
||||
{
|
||||
s.Append(':');
|
||||
s.Append(this.request.ToString());
|
||||
}
|
||||
|
||||
public override EventId EventId => this.request.EventId;
|
||||
|
||||
public override Netherite.InstanceQuery InstanceQuery => this.request.InstanceQuery;
|
||||
|
||||
public override async Task OnQueryCompleteAsync(IAsyncEnumerable<OrchestrationState> result, Partition partition)
|
||||
{
|
||||
partition.Assert(this.request.Phase == ClientRequestEventWithQuery.ProcessingPhase.Query);
|
||||
|
||||
await this.request.OnQueryCompleteAsync(result, partition);
|
||||
|
||||
var again = (ClientRequestEventWithQuery)this.request.Clone();
|
||||
|
||||
again.NextInputQueuePosition = 0; // this event is no longer considered an external event
|
||||
|
||||
again.Phase = ClientRequestEventWithQuery.ProcessingPhase.Confirm;
|
||||
|
||||
partition.SubmitInternalEvent(again);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,74 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Runtime.Serialization;
|
||||
|
||||
[DataContract]
|
||||
class ReassemblyState : TrackedObject
|
||||
{
|
||||
[DataMember]
|
||||
public Dictionary<string, List<PartitionEventFragment>> Fragments { get; private set; } = new Dictionary<string, List<PartitionEventFragment>>();
|
||||
|
||||
[IgnoreDataMember]
|
||||
public override TrackedObjectKey Key => new TrackedObjectKey(TrackedObjectKey.TrackedObjectType.Reassembly);
|
||||
public override string ToString()
|
||||
{
|
||||
return $"Reassembly ({this.Fragments.Count} pending)";
|
||||
}
|
||||
|
||||
public override void Process(PartitionEventFragment evt, EffectTracker effects)
|
||||
{
|
||||
// stores fragments until the last one is received
|
||||
var originalEventString = evt.OriginalEventId.ToString();
|
||||
if (evt.IsLast)
|
||||
{
|
||||
evt.ReassembledEvent = FragmentationAndReassembly.Reassemble<PartitionEvent>(this.Fragments[originalEventString], evt);
|
||||
|
||||
this.Partition.EventDetailTracer?.TraceEventProcessingDetail($"Reassembled {evt.ReassembledEvent}");
|
||||
|
||||
this.Fragments.Remove(originalEventString);
|
||||
|
||||
switch (evt.ReassembledEvent)
|
||||
{
|
||||
case PartitionUpdateEvent updateEvent:
|
||||
updateEvent.DetermineEffects(effects);
|
||||
break;
|
||||
|
||||
case PartitionReadEvent readEvent:
|
||||
this.Partition.SubmitInternalEvent(readEvent);
|
||||
break;
|
||||
|
||||
case PartitionQueryEvent queryEvent:
|
||||
this.Partition.SubmitInternalEvent(queryEvent);
|
||||
break;
|
||||
|
||||
default:
|
||||
throw new InvalidCastException("Could not cast to neither PartitionReadEvent nor PartitionUpdateEvent");
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (!this.Fragments.TryGetValue(originalEventString, out var list))
|
||||
{
|
||||
this.Fragments[originalEventString] = list = new List<PartitionEventFragment>();
|
||||
}
|
||||
list.Add(evt);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,343 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Linq;
|
||||
using System.Runtime.Serialization;
|
||||
using DurableTask.Core;
|
||||
using DurableTask.Core.History;
|
||||
using DurableTask.Netherite.Scaling;
|
||||
|
||||
[DataContract]
|
||||
class SessionsState : TrackedObject, TransportAbstraction.IDurabilityListener
|
||||
{
|
||||
[DataMember]
|
||||
public Dictionary<string, Session> Sessions { get; private set; } = new Dictionary<string, Session>();
|
||||
|
||||
[DataMember]
|
||||
public long SequenceNumber { get; set; }
|
||||
|
||||
[DataContract]
|
||||
internal class Session
|
||||
{
|
||||
[DataMember]
|
||||
public long SessionId { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public long BatchStartPosition { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public List<TaskMessage> Batch { get; set; }
|
||||
|
||||
[DataMember]
|
||||
public bool ForceNewExecution { get; set; }
|
||||
|
||||
[IgnoreDataMember]
|
||||
public OrchestrationMessageBatch CurrentBatch { get; set; }
|
||||
}
|
||||
|
||||
[DataMember]
|
||||
public Dictionary<string, BatchProcessed> StepsAwaitingPersistence { get; private set; } = new Dictionary<string, BatchProcessed>();
|
||||
|
||||
[IgnoreDataMember]
|
||||
public HashSet<OrchestrationMessageBatch> PendingMessageBatches { get; set; } = new HashSet<OrchestrationMessageBatch>();
|
||||
|
||||
[IgnoreDataMember]
|
||||
public override TrackedObjectKey Key => new TrackedObjectKey(TrackedObjectKey.TrackedObjectType.Sessions);
|
||||
|
||||
public static string GetWorkItemId(uint partition, long session, long position) => $"{partition:D2}-S{session}:{position}";
|
||||
|
||||
|
||||
public override void OnRecoveryCompleted()
|
||||
{
|
||||
// start work items for all sessions
|
||||
foreach (var kvp in this.Sessions)
|
||||
{
|
||||
new OrchestrationMessageBatch(kvp.Key, kvp.Value, this.Partition);
|
||||
}
|
||||
|
||||
// handle all steps that were awaiting persistence
|
||||
foreach(var kvp in this.StepsAwaitingPersistence)
|
||||
{
|
||||
this.ConfirmDurable(kvp.Value);
|
||||
}
|
||||
}
|
||||
|
||||
public override void UpdateLoadInfo(PartitionLoadInfo info)
|
||||
{
|
||||
info.WorkItems += this.Sessions.Count;
|
||||
double now = this.Partition.CurrentTimeMs;
|
||||
info.WorkItemLatencyMs = (long) this.Sessions.Values.Select(session => session.CurrentBatch?.WaitTimeMs(now) ?? 0).DefaultIfEmpty().Max();
|
||||
}
|
||||
|
||||
public override string ToString()
|
||||
{
|
||||
return $"Sessions ({this.Sessions.Count} pending) next={this.SequenceNumber:D6}";
|
||||
}
|
||||
|
||||
string GetSessionPosition(Session session) => $"{this.Partition.PartitionId:D2}-S{session.SessionId}:{session.BatchStartPosition + session.Batch.Count}";
|
||||
|
||||
|
||||
void AddMessageToSession(TaskMessage message, bool isReplaying)
|
||||
{
|
||||
string instanceId = message.OrchestrationInstance.InstanceId;
|
||||
bool forceNewExecution = message.Event is ExecutionStartedEvent;
|
||||
|
||||
if (this.Sessions.TryGetValue(instanceId, out var session) && !forceNewExecution)
|
||||
{
|
||||
// A session for this instance already exists, so a work item is in progress already.
|
||||
// We don't need to schedule a work item because we'll notice the new messages when it completes.
|
||||
this.Partition.EventTraceHelper.TraceTaskMessageReceived(message, this.GetSessionPosition(session));
|
||||
session.Batch.Add(message);
|
||||
}
|
||||
else
|
||||
{
|
||||
this.Sessions[instanceId] = session = new Session()
|
||||
{
|
||||
SessionId = this.SequenceNumber++,
|
||||
Batch = new List<TaskMessage>(),
|
||||
BatchStartPosition = 0,
|
||||
ForceNewExecution = forceNewExecution,
|
||||
};
|
||||
|
||||
this.Partition.EventTraceHelper.TraceTaskMessageReceived(message, this.GetSessionPosition(session));
|
||||
session.Batch.Add(message);
|
||||
if (!isReplaying) // during replay, we don't start work items until end of recovery
|
||||
{
|
||||
new OrchestrationMessageBatch(instanceId, session, this.Partition);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void AddMessagesToSession(string instanceId, IEnumerable<TaskMessage> messages, bool isReplaying)
|
||||
{
|
||||
int? forceNewExecution = FindLastExecutionStartedEvent(messages);
|
||||
|
||||
if (this.Sessions.TryGetValue(instanceId, out var session) && forceNewExecution == null)
|
||||
{
|
||||
// A session for this instance already exists, so a work item is in progress already.
|
||||
// We don't need to schedule a work item because we'll notice the new messages
|
||||
// when the previous work item completes.
|
||||
foreach(var message in messages)
|
||||
{
|
||||
this.Partition.EventTraceHelper.TraceTaskMessageReceived(message, this.GetSessionPosition(session));
|
||||
session.Batch.Add(message);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (forceNewExecution.HasValue)
|
||||
{
|
||||
// the new instance replaces whatever state the old instance was in
|
||||
// since our transport is exactly once and in order, we do not save the old messages
|
||||
// but "consider them delivered" to the old instance which is then replaced
|
||||
foreach (var taskMessage in messages.Take(forceNewExecution.Value))
|
||||
{
|
||||
this.Partition.EventTraceHelper.TraceTaskMessageDiscarded(taskMessage, "message bound for an instance that was replaced", "");
|
||||
}
|
||||
|
||||
messages = messages.Skip(forceNewExecution.Value);
|
||||
}
|
||||
|
||||
// Create a new session
|
||||
this.Sessions[instanceId] = session = new Session()
|
||||
{
|
||||
SessionId = this.SequenceNumber++,
|
||||
Batch = new List<TaskMessage>(),
|
||||
BatchStartPosition = 0,
|
||||
ForceNewExecution = forceNewExecution.HasValue,
|
||||
};
|
||||
|
||||
foreach (var message in messages)
|
||||
{
|
||||
this.Partition.EventTraceHelper.TraceTaskMessageReceived(message, this.GetSessionPosition(session));
|
||||
session.Batch.Add(message);
|
||||
}
|
||||
|
||||
if (!isReplaying) // we don't start work items until end of recovery
|
||||
{
|
||||
new OrchestrationMessageBatch(instanceId, session, this.Partition);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static int? FindLastExecutionStartedEvent(IEnumerable<TaskMessage> messages)
|
||||
{
|
||||
int? lastOccurence = null;
|
||||
int position = 0;
|
||||
foreach (TaskMessage taskMessage in messages)
|
||||
{
|
||||
if (taskMessage.Event is ExecutionStartedEvent)
|
||||
{
|
||||
lastOccurence = position;
|
||||
}
|
||||
position++;
|
||||
}
|
||||
return lastOccurence;
|
||||
}
|
||||
|
||||
public void Process(TaskMessagesReceived evt, EffectTracker effects)
|
||||
{
|
||||
// queues task message (from another partition) in a new or existing session
|
||||
foreach (var group in evt.TaskMessages
|
||||
.GroupBy(tm => tm.OrchestrationInstance.InstanceId))
|
||||
{
|
||||
this.AddMessagesToSession(group.Key, group, effects.IsReplaying);
|
||||
}
|
||||
}
|
||||
|
||||
public void Process(RemoteActivityResultReceived evt, EffectTracker effects)
|
||||
{
|
||||
// queues task message (from another partition) in a new or existing session
|
||||
this.AddMessageToSession(evt.Result, effects.IsReplaying);
|
||||
}
|
||||
|
||||
public void Process(ClientTaskMessagesReceived evt, EffectTracker effects)
|
||||
{
|
||||
// queues task message (from a client) in a new or existing session
|
||||
var instanceId = evt.TaskMessages[0].OrchestrationInstance.InstanceId;
|
||||
this.AddMessagesToSession(instanceId, evt.TaskMessages, effects.IsReplaying);
|
||||
}
|
||||
|
||||
public void Process(TimerFired timerFired, EffectTracker effects)
|
||||
{
|
||||
// queues a timer fired message in a session
|
||||
this.AddMessageToSession(timerFired.TaskMessage, effects.IsReplaying);
|
||||
}
|
||||
|
||||
public void Process(ActivityCompleted activityCompleted, EffectTracker effects)
|
||||
{
|
||||
// queues an activity-completed message in a session
|
||||
this.AddMessageToSession(activityCompleted.Response, effects.IsReplaying);
|
||||
}
|
||||
|
||||
public void Process(CreationRequestReceived creationRequestReceived, EffectTracker effects)
|
||||
{
|
||||
// queues the execution started message
|
||||
this.AddMessageToSession(creationRequestReceived.TaskMessage, effects.IsReplaying);
|
||||
}
|
||||
|
||||
public void Process(DeletionRequestReceived deletionRequestReceived, EffectTracker effects)
|
||||
{
|
||||
// removing the session means that all pending messages will be deleted also.
|
||||
this.Sessions.Remove(deletionRequestReceived.InstanceId);
|
||||
}
|
||||
|
||||
public void Process(PurgeBatchIssued purgeBatchIssued, EffectTracker effects)
|
||||
{
|
||||
foreach (string instanceId in purgeBatchIssued.Purged)
|
||||
{
|
||||
// removing the session means that all pending messages will be deleted also.
|
||||
this.Sessions.Remove(instanceId);
|
||||
}
|
||||
}
|
||||
|
||||
public void ConfirmDurable(Event evt)
|
||||
{
|
||||
var evtCopy = (BatchProcessed) ((BatchProcessed) evt).Clone();
|
||||
evtCopy.IsPersisted = true;
|
||||
this.Partition.SubmitInternalEvent(evtCopy);
|
||||
}
|
||||
|
||||
public void Process(BatchProcessed evt, EffectTracker effects)
|
||||
{
|
||||
// if speculation is disabled,
|
||||
if (effects.Partition.Settings.PersistStepsFirst)
|
||||
{
|
||||
if (!evt.IsPersisted)
|
||||
{
|
||||
// we do not process this event right away
|
||||
// but persist it first, and then submit it again, before processing it.
|
||||
this.StepsAwaitingPersistence.Add(evt.WorkItemId, evt);
|
||||
DurabilityListeners.Register(evt, this);
|
||||
return;
|
||||
}
|
||||
else
|
||||
{
|
||||
this.StepsAwaitingPersistence.Remove(evt.WorkItemId);
|
||||
}
|
||||
}
|
||||
|
||||
// updates the session and other state
|
||||
|
||||
// our instance may already be obsolete if it has been forcefully replaced.
|
||||
// This can manifest as the instance having disappeared, or as the current instance having
|
||||
// a different session id
|
||||
if (!this.Sessions.TryGetValue(evt.InstanceId, out var session) || session.SessionId != evt.SessionId)
|
||||
{
|
||||
this.Partition.EventTraceHelper.TraceOrchestrationWorkItemDiscarded(evt);
|
||||
return;
|
||||
};
|
||||
|
||||
if (evt.ActivityMessages?.Count > 0)
|
||||
{
|
||||
effects.Add(TrackedObjectKey.Activities);
|
||||
}
|
||||
|
||||
if (evt.TimerMessages?.Count > 0)
|
||||
{
|
||||
effects.Add(TrackedObjectKey.Timers);
|
||||
}
|
||||
|
||||
if (evt.RemoteMessages?.Count > 0)
|
||||
{
|
||||
effects.Add(TrackedObjectKey.Outbox);
|
||||
}
|
||||
|
||||
// deliver orchestrator messages destined for this partition directly to the relevant session(s)
|
||||
if (evt.LocalMessages?.Count > 0)
|
||||
{
|
||||
foreach (var group in evt.LocalMessages.GroupBy(tm => tm.OrchestrationInstance.InstanceId))
|
||||
{
|
||||
this.AddMessagesToSession(group.Key, group, effects.IsReplaying);
|
||||
}
|
||||
}
|
||||
|
||||
if (evt.State != null)
|
||||
{
|
||||
effects.Add(TrackedObjectKey.Instance(evt.InstanceId));
|
||||
effects.Add(TrackedObjectKey.History(evt.InstanceId));
|
||||
}
|
||||
|
||||
// remove processed messages from this batch
|
||||
effects.Partition.Assert(session != null);
|
||||
effects.Partition.Assert(session.SessionId == evt.SessionId);
|
||||
effects.Partition.Assert(session.BatchStartPosition == evt.BatchStartPosition);
|
||||
session.Batch.RemoveRange(0, evt.BatchLength);
|
||||
session.BatchStartPosition += evt.BatchLength;
|
||||
|
||||
this.StartNewBatchIfNeeded(session, effects, evt.InstanceId, effects.IsReplaying);
|
||||
}
|
||||
|
||||
void StartNewBatchIfNeeded(Session session, EffectTracker effects, string instanceId, bool inRecovery)
|
||||
{
|
||||
if (session.Batch.Count == 0)
|
||||
{
|
||||
// no more pending messages for this instance, so we delete the session.
|
||||
this.Sessions.Remove(instanceId);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (!inRecovery) // we don't start work items until end of recovery
|
||||
{
|
||||
// there are more messages. Start another work item.
|
||||
new OrchestrationMessageBatch(instanceId, session, this.Partition);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,153 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.Serialization;
|
||||
using DurableTask.Core;
|
||||
using DurableTask.Core.Common;
|
||||
using DurableTask.Core.History;
|
||||
using DurableTask.Netherite.Scaling;
|
||||
|
||||
[DataContract]
|
||||
class TimersState : TrackedObject
|
||||
{
|
||||
[DataMember]
|
||||
public Dictionary<long, (DateTime, TaskMessage)> PendingTimers { get; private set; } = new Dictionary<long, (DateTime, TaskMessage)>();
|
||||
|
||||
[DataMember]
|
||||
public long SequenceNumber { get; set; }
|
||||
|
||||
[IgnoreDataMember]
|
||||
public override TrackedObjectKey Key => new TrackedObjectKey(TrackedObjectKey.TrackedObjectType.Timers);
|
||||
|
||||
public override string ToString()
|
||||
{
|
||||
return $"Timers ({this.PendingTimers.Count} pending) next={this.SequenceNumber:D6}";
|
||||
}
|
||||
|
||||
public override void OnRecoveryCompleted()
|
||||
{
|
||||
// restore the pending timers
|
||||
foreach (var kvp in this.PendingTimers)
|
||||
{
|
||||
this.Schedule(kvp.Key, kvp.Value.Item1, kvp.Value.Item2);
|
||||
}
|
||||
}
|
||||
|
||||
// how long before the scheduled time the ScalingMonitor should scale up from zero
|
||||
static readonly TimeSpan WakeupInAdvance = TimeSpan.FromSeconds(20);
|
||||
|
||||
public override void UpdateLoadInfo(PartitionLoadInfo info)
|
||||
{
|
||||
info.Timers = this.PendingTimers.Count;
|
||||
|
||||
if (info.Timers > 0)
|
||||
{
|
||||
info.Wakeup = this.PendingTimers.Select(kvp => kvp.Value.Item1).Min() - WakeupInAdvance;
|
||||
}
|
||||
else
|
||||
{
|
||||
info.Wakeup = null;
|
||||
}
|
||||
}
|
||||
|
||||
void Schedule(long timerId, DateTime due, TaskMessage message)
|
||||
{
|
||||
TimerFired expirationEvent = new TimerFired()
|
||||
{
|
||||
PartitionId = this.Partition.PartitionId,
|
||||
TimerId = timerId,
|
||||
TaskMessage = message,
|
||||
Due = due,
|
||||
};
|
||||
|
||||
this.Partition.EventDetailTracer?.TraceEventProcessingDetail($"Scheduled {message} due at {expirationEvent.Due:o}, id={expirationEvent.EventIdString}");
|
||||
this.Partition.PendingTimers.Schedule(expirationEvent.Due, expirationEvent);
|
||||
}
|
||||
|
||||
static DateTime GetDueTime(TaskMessage message)
|
||||
{
|
||||
if (message.Event is TimerFiredEvent timerFiredEvent)
|
||||
{
|
||||
return timerFiredEvent.FireAt;
|
||||
}
|
||||
else if (Entities.IsDelayedEntityMessage(message, out DateTime due))
|
||||
{
|
||||
return due;
|
||||
}
|
||||
else if (message.Event is ExecutionStartedEvent executionStartedEvent && executionStartedEvent.ScheduledStartTime.HasValue)
|
||||
{
|
||||
return executionStartedEvent.ScheduledStartTime.Value;
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new ArgumentException(nameof(message), "unhandled event type");
|
||||
}
|
||||
}
|
||||
|
||||
public void Process(TimerFired evt, EffectTracker effects)
|
||||
{
|
||||
// removes the entry for the pending timer, and then adds it to the sessions queue
|
||||
this.PendingTimers.Remove(evt.TimerId);
|
||||
}
|
||||
|
||||
public void Process(BatchProcessed evt, EffectTracker effects)
|
||||
{
|
||||
// starts new timers as specified by the batch
|
||||
foreach (var t in evt.TimerMessages)
|
||||
{
|
||||
var timerId = this.SequenceNumber++;
|
||||
var due = GetDueTime(t);
|
||||
this.PendingTimers.Add(timerId, (due, t));
|
||||
|
||||
if (!effects.IsReplaying)
|
||||
{
|
||||
this.Schedule(timerId, due, t);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void Process(TaskMessagesReceived evt, EffectTracker effects)
|
||||
{
|
||||
// starts new timers as specified by the batch
|
||||
foreach (var t in evt.DelayedTaskMessages)
|
||||
{
|
||||
var timerId = this.SequenceNumber++;
|
||||
var due = GetDueTime(t);
|
||||
this.PendingTimers.Add(timerId, (due, t));
|
||||
|
||||
if (!effects.IsReplaying)
|
||||
{
|
||||
this.Schedule(timerId, due, t);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void Process(CreationRequestReceived creationRequestReceived, EffectTracker effects)
|
||||
{
|
||||
// starts a new timer for the execution started event
|
||||
var timerId = this.SequenceNumber++;
|
||||
var due = GetDueTime(creationRequestReceived.TaskMessage);
|
||||
this.PendingTimers.Add(timerId, (due, creationRequestReceived.TaskMessage));
|
||||
|
||||
if (!effects.IsReplaying)
|
||||
{
|
||||
this.Schedule(timerId, due, creationRequestReceived.TaskMessage);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,170 @@
|
|||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite.Scaling
|
||||
{
|
||||
using Microsoft.Azure.Cosmos.Table;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
class AzureLoadMonitorTable : ILoadMonitorService
|
||||
{
|
||||
readonly CloudTable table;
|
||||
readonly string taskHubName;
|
||||
|
||||
public AzureLoadMonitorTable(string connectionString, string tableName, string taskHubName)
|
||||
{
|
||||
var account = CloudStorageAccount.Parse(connectionString);
|
||||
this.table = account.CreateCloudTableClient().GetTableReference(tableName);
|
||||
this.taskHubName = taskHubName;
|
||||
}
|
||||
|
||||
public TimeSpan PublishInterval => TimeSpan.FromSeconds(10);
|
||||
|
||||
public async Task DeleteIfExistsAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (! await this.table.ExistsAsync().ConfigureAwait(false))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var query = new TableQuery<PartitionInfoEntity>().Where(TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.Equal, this.taskHubName));
|
||||
TableContinuationToken continuationToken = null;
|
||||
do
|
||||
{
|
||||
var batch = await this.table.ExecuteQuerySegmentedAsync<PartitionInfoEntity>(query, continuationToken, null, null, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (batch.Count() > 0)
|
||||
{
|
||||
// delete all entities in this batch. Max partition number is 32 so it always fits.
|
||||
TableBatchOperation tableBatch = new TableBatchOperation();
|
||||
|
||||
foreach (var e in batch)
|
||||
{
|
||||
tableBatch.Add(TableOperation.Delete(e));
|
||||
}
|
||||
|
||||
await this.table.ExecuteBatchAsync(tableBatch).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
while (continuationToken != null);
|
||||
}
|
||||
|
||||
public async Task CreateIfNotExistsAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (await this.table.ExistsAsync().ConfigureAwait(false))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await this.table.CreateAsync().ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public Task PublishAsync(Dictionary<uint, PartitionLoadInfo> info, CancellationToken cancellationToken)
|
||||
{
|
||||
TableBatchOperation tableBatch = new TableBatchOperation();
|
||||
foreach(var kvp in info)
|
||||
{
|
||||
tableBatch.Add(TableOperation.InsertOrReplace(new PartitionInfoEntity(this.taskHubName, kvp.Key, kvp.Value)));
|
||||
}
|
||||
return this.table.ExecuteBatchAsync(tableBatch, null, null, cancellationToken);
|
||||
}
|
||||
|
||||
public async Task<Dictionary<uint, PartitionLoadInfo>> QueryAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var query = new TableQuery<PartitionInfoEntity>().Where(TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.Equal, this.taskHubName));
|
||||
TableContinuationToken continuationToken = null;
|
||||
Dictionary<uint, PartitionLoadInfo> result = new Dictionary<uint, PartitionLoadInfo>();
|
||||
do
|
||||
{
|
||||
var batch = await this.table.ExecuteQuerySegmentedAsync<PartitionInfoEntity>(query, continuationToken, null, null, cancellationToken).ConfigureAwait(false);
|
||||
foreach (var e in batch)
|
||||
{
|
||||
result.Add(e.PartitionId, new PartitionLoadInfo()
|
||||
{
|
||||
WorkItems = e.WorkItems,
|
||||
Activities = e.Activities,
|
||||
Timers = e.Timers,
|
||||
Requests = e.Requests,
|
||||
Wakeup = e.NextTimer,
|
||||
Outbox = e.Outbox,
|
||||
InputQueuePosition = e.InputQueuePosition,
|
||||
CommitLogPosition = e.CommitLogPosition,
|
||||
ActivityLatencyMs = e.ActivityLatencyMs,
|
||||
WorkItemLatencyMs = e.WorkItemLatencyMs,
|
||||
WorkerId = e.WorkerId,
|
||||
LatencyTrend = e.LatencyTrend,
|
||||
MissRate = e.MissRate,
|
||||
});
|
||||
}
|
||||
}
|
||||
while (continuationToken != null);
|
||||
return result;
|
||||
}
|
||||
|
||||
public class PartitionInfoEntity : TableEntity
|
||||
{
|
||||
public int WorkItems { get; set; }
|
||||
public int Activities { get; set; }
|
||||
public int Timers { get; set; }
|
||||
public int Requests { get; set; }
|
||||
public int Outbox { get; set; }
|
||||
public DateTime? NextTimer { get; set; }
|
||||
public long InputQueuePosition { get; set; }
|
||||
public long CommitLogPosition { get; set; }
|
||||
public long ActivityLatencyMs { get; set; }
|
||||
public long WorkItemLatencyMs { get; set; }
|
||||
public string WorkerId { get; set; }
|
||||
public string LatencyTrend { get; set; }
|
||||
public double MissRate { get; set; }
|
||||
|
||||
public PartitionInfoEntity()
|
||||
{
|
||||
}
|
||||
|
||||
// constructor for creating a deletion query; only the partition and row keys matter
|
||||
public PartitionInfoEntity(string taskHubName, uint partitionId)
|
||||
: base(taskHubName, partitionId.ToString("D2"))
|
||||
{
|
||||
this.ETag = "*"; // no conditions when deleting
|
||||
}
|
||||
|
||||
// constructor for updating load information
|
||||
public PartitionInfoEntity(string taskHubName, uint partitionId, PartitionLoadInfo info)
|
||||
: base(taskHubName, partitionId.ToString("D2"))
|
||||
{
|
||||
this.WorkItems = info.WorkItems;
|
||||
this.Activities = info.Activities;
|
||||
this.Timers = info.Timers;
|
||||
this.Requests = info.Requests;
|
||||
this.NextTimer = info.Wakeup;
|
||||
this.Outbox = info.Outbox;
|
||||
this.InputQueuePosition = info.InputQueuePosition;
|
||||
this.CommitLogPosition = info.CommitLogPosition;
|
||||
this.ActivityLatencyMs = info.ActivityLatencyMs;
|
||||
this.WorkItemLatencyMs = info.WorkItemLatencyMs;
|
||||
this.WorkerId = info.WorkerId;
|
||||
this.LatencyTrend = info.LatencyTrend;
|
||||
this.MissRate = info.MissRate;
|
||||
|
||||
this.ETag = "*"; // no conditions when inserting, replace existing
|
||||
}
|
||||
|
||||
public string TaskHubName => this.PartitionKey;
|
||||
public uint PartitionId => uint.Parse(this.RowKey);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,55 @@
|
|||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite.Scaling
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
/// <summary>
|
||||
/// An interface for the load monitor service.
|
||||
/// </summary>
|
||||
public interface ILoadMonitorService
|
||||
{
|
||||
/// <summary>
|
||||
/// Publish the load of a partition to the service.
|
||||
/// </summary>
|
||||
/// <param name="loadInfo">A collection of load information for partitions</param>
|
||||
/// <param name="cancellationToken">A cancellation token</param>
|
||||
/// <returns>A task indicating completion</returns>
|
||||
Task PublishAsync(Dictionary<uint, PartitionLoadInfo> loadInfo, CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Delete all load information for a taskhub.
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">A cancellation token</param>
|
||||
/// <returns>A task indicating completion</returns>
|
||||
Task DeleteIfExistsAsync(CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Prepare the service for a taskhub.
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">A cancellation token</param>
|
||||
/// <returns>A task indicating completion</returns>
|
||||
Task CreateIfNotExistsAsync(CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Query all load information for a taskhub.
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">A cancellation token</param>
|
||||
/// <returns>A task returning a dictionary with load information for the partitions</returns>
|
||||
Task<Dictionary<uint, PartitionLoadInfo>> QueryAsync(CancellationToken cancellationToken);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,85 @@
|
|||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite.Scaling
|
||||
{
|
||||
using Microsoft.Extensions.Logging;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
class LoadPublisher : BatchWorker<(uint, PartitionLoadInfo)>
|
||||
{
|
||||
readonly ILoadMonitorService service;
|
||||
readonly ILogger logger;
|
||||
|
||||
// we are pushing the aggregated load information on a somewhat slower interval
|
||||
public static TimeSpan AggregatePublishInterval = TimeSpan.FromSeconds(15);
|
||||
readonly CancellationTokenSource cancelWait = new CancellationTokenSource();
|
||||
|
||||
public LoadPublisher(ILoadMonitorService service, ILogger logger) : base(nameof(LoadPublisher))
|
||||
{
|
||||
this.service = service;
|
||||
this.logger = logger;
|
||||
this.cancelWait = new CancellationTokenSource();
|
||||
}
|
||||
|
||||
public void Flush()
|
||||
{
|
||||
this.cancelWait.Cancel(); // so that we don't have to wait the whole delay
|
||||
}
|
||||
|
||||
protected override async Task Process(IList<(uint, PartitionLoadInfo)> batch)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (batch.Count != 0)
|
||||
{
|
||||
var latestForEachPartition = new Dictionary<uint, PartitionLoadInfo>();
|
||||
|
||||
foreach (var (partitionId, info) in batch)
|
||||
{
|
||||
latestForEachPartition[partitionId] = info;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await this.service.PublishAsync(latestForEachPartition, this.cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// o.k. during shutdown
|
||||
}
|
||||
catch
|
||||
{
|
||||
// we swallow exceptions so we can tolerate temporary Azure storage errors
|
||||
// TODO log
|
||||
}
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await Task.Delay(AggregatePublishInterval, this.cancelWait.Token).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
}
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
this.logger.LogWarning("Could not publish load {exception}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,119 @@
|
|||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
namespace DurableTask.Netherite.Scaling
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
|
||||
/// <summary>
|
||||
/// Reported load information about a specific partition.
|
||||
/// </summary>
|
||||
public class PartitionLoadInfo
|
||||
{
|
||||
/// <summary>
|
||||
/// The number of orchestration work items waiting to be processed.
|
||||
/// </summary>
|
||||
public int WorkItems { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The number of activities that are waiting to be processed.
|
||||
/// </summary>
|
||||
public int Activities { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The number of timers that are waiting to fire.
|
||||
/// </summary>
|
||||
public int Timers { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The number of client requests waiting to be processed.
|
||||
/// </summary>
|
||||
public int Requests { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The number of work items that have messages waiting to be sent.
|
||||
/// </summary>
|
||||
public int Outbox { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The next time on which to wake up.
|
||||
/// </summary>
|
||||
public DateTime? Wakeup { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The input queue position of this partition, which is the next expected EventHubs sequence number.
|
||||
/// </summary>
|
||||
public long InputQueuePosition { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The commit log position of this partition.
|
||||
/// </summary>
|
||||
public long CommitLogPosition { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The latency of the activity queue.
|
||||
/// </summary>
|
||||
public long ActivityLatencyMs { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The latency of the work item queue.
|
||||
/// </summary>
|
||||
public long WorkItemLatencyMs { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The worker id of the host that is currently running this partition.
|
||||
/// </summary>
|
||||
public string WorkerId { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// A string encoding of the latency trend.
|
||||
/// </summary>
|
||||
public string LatencyTrend { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Percentage of message batches that miss in the cache.
|
||||
/// </summary>
|
||||
public double MissRate { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The character representing idle load.
|
||||
/// </summary>
|
||||
public const char Idle = 'I';
|
||||
|
||||
/// <summary>
|
||||
/// The character representing low latency.
|
||||
/// </summary>
|
||||
public const char LowLatency = 'L';
|
||||
|
||||
/// <summary>
|
||||
/// The character representing medium latency.
|
||||
/// </summary>
|
||||
public const char MediumLatency = 'M';
|
||||
|
||||
/// <summary>
|
||||
/// The character representing high latency.
|
||||
/// </summary>
|
||||
public const char HighLatency = 'H';
|
||||
|
||||
/// <summary>
|
||||
/// All of the latency category characters in order
|
||||
/// </summary>
|
||||
public static char[] LatencyCategories = new char[] { Idle, LowLatency, MediumLatency, HighLatency };
|
||||
|
||||
/// <summary>
|
||||
/// The maximum length of the latency trend
|
||||
/// </summary>
|
||||
public static int LatencyTrendLength = 5;
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite.Scaling
|
||||
{
|
||||
/// <summary>
|
||||
/// Possible scale actions for durable task hub.
|
||||
/// </summary>
|
||||
public enum ScaleAction
|
||||
{
|
||||
/// <summary>
|
||||
/// Do not add or remove workers.
|
||||
/// </summary>
|
||||
None = 0,
|
||||
|
||||
/// <summary>
|
||||
/// Add workers to the current task hub.
|
||||
/// </summary>
|
||||
AddWorker,
|
||||
|
||||
/// <summary>
|
||||
/// Remove workers from the current task hub.
|
||||
/// </summary>
|
||||
RemoveWorker
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
namespace DurableTask.Netherite.Scaling
|
||||
{
|
||||
using System;
|
||||
|
||||
/// <summary>
|
||||
/// Represents a scale recommendation for the task hub given the current performance metrics.
|
||||
/// </summary>
|
||||
public class ScaleRecommendation : EventArgs
|
||||
{
|
||||
/// <summary>
|
||||
/// Constructs a scale recommendation.
|
||||
/// </summary>
|
||||
/// <param name="scaleAction">The scale action.</param>
|
||||
/// <param name="keepWorkersAlive">Whether to keep workers alive.</param>
|
||||
/// <param name="reason">Text describing the reason for the recommendation.</param>
|
||||
public ScaleRecommendation(ScaleAction scaleAction, bool keepWorkersAlive, string reason)
|
||||
{
|
||||
this.Action = scaleAction;
|
||||
this.KeepWorkersAlive = keepWorkersAlive;
|
||||
this.Reason = reason;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the recommended scale action for the current task hub.
|
||||
/// </summary>
|
||||
public ScaleAction Action { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets a recommendation about whether to keep existing task hub workers alive.
|
||||
/// </summary>
|
||||
public bool KeepWorkersAlive { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets text describing why a particular scale action was recommended.
|
||||
/// </summary>
|
||||
public string Reason { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets a string description of the current <see cref="ScaleRecommendation"/> object.
|
||||
/// </summary>
|
||||
/// <returns>A string description useful for diagnostics.</returns>
|
||||
public override string ToString()
|
||||
{
|
||||
return $"{nameof(this.Action)}: {this.Action}, {nameof(this.KeepWorkersAlive)}: {this.KeepWorkersAlive}, {nameof(this.Reason)}: {this.Reason}";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,177 @@
|
|||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
namespace DurableTask.Netherite.Scaling
|
||||
{
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using DurableTask.Netherite.EventHubs;
|
||||
|
||||
/// <summary>
|
||||
/// Monitors the performance of the Netherite backend and makes scaling decisions.
|
||||
/// </summary>
|
||||
public class ScalingMonitor
|
||||
{
|
||||
readonly string storageConnectionString;
|
||||
readonly string eventHubsConnectionString;
|
||||
readonly string partitionLoadTableName;
|
||||
readonly string taskHubName;
|
||||
readonly TransportConnectionString.TransportChoices configuredTransport;
|
||||
|
||||
readonly AzureLoadMonitorTable table;
|
||||
|
||||
/// <summary>
|
||||
/// Creates an instance of the scaling monitor, with the given parameters.
|
||||
/// </summary>
|
||||
/// <param name="storageConnectionString">The storage connection string.</param>
|
||||
/// <param name="eventHubsConnectionString">The connection string for the transport layer.</param>
|
||||
/// <param name="partitionLoadTableName">The name of the storage table with the partition load information.</param>
|
||||
/// <param name="taskHubName">The name of the taskhub.</param>
|
||||
public ScalingMonitor(string storageConnectionString, string eventHubsConnectionString, string partitionLoadTableName, string taskHubName)
|
||||
{
|
||||
this.storageConnectionString = storageConnectionString;
|
||||
this.eventHubsConnectionString = eventHubsConnectionString;
|
||||
this.partitionLoadTableName = partitionLoadTableName;
|
||||
this.taskHubName = taskHubName;
|
||||
|
||||
TransportConnectionString.Parse(eventHubsConnectionString, out _, out this.configuredTransport, out _);
|
||||
|
||||
this.table = new AzureLoadMonitorTable(storageConnectionString, partitionLoadTableName, taskHubName);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Makes a scale recommendation.
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
public async Task<ScaleRecommendation> GetScaleRecommendation(int workerCount)
|
||||
{
|
||||
Dictionary<uint, PartitionLoadInfo> loadInformation = await this.table.QueryAsync(CancellationToken.None).ConfigureAwait(false);
|
||||
|
||||
bool taskHubIsIdle = await this.TaskHubIsIdleAsync(loadInformation).ConfigureAwait(false);
|
||||
|
||||
if (workerCount == 0 && !taskHubIsIdle)
|
||||
{
|
||||
return new ScaleRecommendation(ScaleAction.AddWorker, keepWorkersAlive: true, reason: "First worker");
|
||||
}
|
||||
|
||||
if (loadInformation.Values.Any(partitionLoadInfo => partitionLoadInfo.LatencyTrend.Length < PartitionLoadInfo.LatencyTrendLength))
|
||||
{
|
||||
return new ScaleRecommendation(ScaleAction.None, keepWorkersAlive: !taskHubIsIdle, reason: "Not enough samples");
|
||||
}
|
||||
|
||||
if (taskHubIsIdle)
|
||||
{
|
||||
return new ScaleRecommendation(
|
||||
scaleAction: workerCount > 0 ? ScaleAction.RemoveWorker : ScaleAction.None,
|
||||
keepWorkersAlive: false,
|
||||
reason: "Task hub is idle");
|
||||
}
|
||||
|
||||
int numberOfSlowPartitions = loadInformation.Values.Count(info => info.LatencyTrend.Last() == PartitionLoadInfo.HighLatency);
|
||||
|
||||
if (workerCount < numberOfSlowPartitions)
|
||||
{
|
||||
// Some partitions are busy, so scale out until workerCount == partitionCount.
|
||||
var partition = loadInformation.First(kvp => kvp.Value.LatencyTrend.Last() == PartitionLoadInfo.HighLatency);
|
||||
return new ScaleRecommendation(
|
||||
ScaleAction.AddWorker,
|
||||
keepWorkersAlive: true,
|
||||
reason: $"High latency in partition {partition.Key}: {partition.Value.LatencyTrend}");
|
||||
}
|
||||
|
||||
int numberOfNonIdlePartitions = loadInformation.Values.Count(info => info.LatencyTrend.Any(c => c != PartitionLoadInfo.Idle));
|
||||
|
||||
if (workerCount > numberOfNonIdlePartitions)
|
||||
{
|
||||
// If the work item queues are idle, scale down to the number of non-idle control queues.
|
||||
return new ScaleRecommendation(
|
||||
ScaleAction.RemoveWorker,
|
||||
keepWorkersAlive: true,
|
||||
reason: $"One or more partitions are idle");
|
||||
}
|
||||
|
||||
// If all queues are operating efficiently, it can be hard to know if we need to reduce the worker count.
|
||||
// We want to avoid the case where a constant trickle of load after a big scale-out prevents scaling back in.
|
||||
// We also want to avoid scaling in unnecessarily when we've reached optimal scale-out. To balance these
|
||||
// goals, we check for low latencies and vote to scale down 10% of the time when we see this. The thought is
|
||||
// that it's a slow scale-in that will get automatically corrected once latencies start increasing again.
|
||||
if (workerCount > 1 && (new Random()).Next(10) == 0)
|
||||
{
|
||||
bool allPartitionsAreFast = !loadInformation.Values.Any(
|
||||
info => info.LatencyTrend.Any(c => c == PartitionLoadInfo.MediumLatency || c == PartitionLoadInfo.HighLatency));
|
||||
|
||||
if (allPartitionsAreFast)
|
||||
{
|
||||
return new ScaleRecommendation(
|
||||
ScaleAction.RemoveWorker,
|
||||
keepWorkersAlive: true,
|
||||
reason: $"All partitions are fast");
|
||||
}
|
||||
}
|
||||
|
||||
// Load exists, but none of our scale filters were triggered, so we assume that the current worker
|
||||
// assignments are close to ideal for the current workload.
|
||||
return new ScaleRecommendation(ScaleAction.None, keepWorkersAlive: true, reason: $"Partition latencies are healthy");
|
||||
}
|
||||
|
||||
async Task<bool> TaskHubIsIdleAsync(Dictionary<uint, PartitionLoadInfo> loadInformation)
|
||||
{
|
||||
// first, check if any of the partitions have queued work or are scheduled to wake up
|
||||
foreach (var p in loadInformation.Values)
|
||||
{
|
||||
if (p.Activities > 0 || p.WorkItems > 0 || p.Requests > 0 || p.Outbox > 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (p.Wakeup.HasValue && p.Wakeup.Value < DateTime.UtcNow + TimeSpan.FromSeconds(10))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// next, check if any of the entries are not current, in the sense that their input queue position
|
||||
// does not match the latest queue position
|
||||
|
||||
long[] positions;
|
||||
|
||||
switch (this.configuredTransport)
|
||||
{
|
||||
case TransportConnectionString.TransportChoices.EventHubs:
|
||||
positions = await EventHubs.EventHubsConnections.GetQueuePositions(this.eventHubsConnectionString, EventHubsTransport.PartitionHubs).ConfigureAwait(false);
|
||||
break;
|
||||
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
|
||||
for (uint i = 0; i < positions.Length; i++)
|
||||
{
|
||||
if (!loadInformation.TryGetValue(i, out var loadInfo))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if (positions[i] > loadInfo.InputQueuePosition)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// we have concluded that there are no pending work items, timers, or unprocessed input queue entries
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,448 @@
|
|||
// ----------------------------------------------------------------------------------
|
||||
// Copyright Microsoft Corporation. All rights reserved.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// ----------------------------------------------------------------------------------
|
||||
|
||||
namespace DurableTask.Netherite.Faster
|
||||
{
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using DurableTask.Core.Common;
|
||||
using FASTER.core;
|
||||
using Microsoft.Azure.Storage;
|
||||
using Microsoft.Azure.Storage.Blob;
|
||||
|
||||
/// <summary>
|
||||
/// A IDevice Implementation that is backed by<see href="https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blob-pageblob-overview">Azure Page Blob</see>.
|
||||
/// This device is slower than a local SSD or HDD, but provides scalability and shared access in the cloud.
|
||||
/// </summary>
|
||||
class AzureStorageDevice : StorageDeviceBase
|
||||
{
|
||||
readonly ConcurrentDictionary<int, BlobEntry> blobs;
|
||||
readonly CloudBlobDirectory blockBlobDirectory;
|
||||
readonly CloudBlobDirectory pageBlobDirectory;
|
||||
readonly string blobName;
|
||||
readonly bool underLease;
|
||||
|
||||
internal IPartitionErrorHandler PartitionErrorHandler { get; private set; }
|
||||
|
||||
// Azure Page Blobs have a fixed sector size of 512 bytes.
|
||||
const uint PAGE_BLOB_SECTOR_SIZE = 512;
|
||||
// Max upload size must be at most 4MB
|
||||
// we use an even smaller value to improve retry/timeout behavior in highly contended situations
|
||||
// Also, this allows us to use aggressive timeouts to kill stragglers
|
||||
const uint MAX_UPLOAD_SIZE = 1024 * 1024;
|
||||
|
||||
const long MAX_PAGEBLOB_SIZE = 512L * 1024 * 1024 * 1024; // set this at 512 GB for now TODO consider implications
|
||||
|
||||
/// <summary>
|
||||
/// Constructs a new AzureStorageDevice instance, backed by Azure Page Blobs
|
||||
/// </summary>
|
||||
/// <param name="blobName">A descriptive name that will be the prefix of all segments created</param>
|
||||
/// <param name="blockBlobDirectory">the directory containing the block blobs</param>
|
||||
/// <param name="pageBlobDirectory">the directory containing the page blobs</param>
|
||||
/// <param name="blobManager">the blob manager handling the leases</param>
|
||||
/// <param name="underLease">whether this device needs to be protected by the lease</param>
|
||||
public AzureStorageDevice(string blobName, CloudBlobDirectory blockBlobDirectory, CloudBlobDirectory pageBlobDirectory, BlobManager blobManager, bool underLease)
|
||||
: base($"{blockBlobDirectory}\\{blobName}", PAGE_BLOB_SECTOR_SIZE, Devices.CAPACITY_UNSPECIFIED)
|
||||
{
|
||||
this.blobs = new ConcurrentDictionary<int, BlobEntry>();
|
||||
this.blockBlobDirectory = blockBlobDirectory;
|
||||
this.pageBlobDirectory = pageBlobDirectory;
|
||||
this.blobName = blobName;
|
||||
this.PartitionErrorHandler = blobManager.PartitionErrorHandler;
|
||||
this.BlobManager = blobManager;
|
||||
this.underLease = underLease;
|
||||
}
|
||||
|
||||
public async Task StartAsync()
|
||||
{
|
||||
this.BlobManager?.StorageTracer?.FasterStorageProgress($"AzureStorageDevice.StartAsync Called target={this.pageBlobDirectory.Prefix}{this.blobName}");
|
||||
|
||||
// list all the blobs representing the segments
|
||||
var prefix = $"{this.blockBlobDirectory.Prefix}{this.blobName}.";
|
||||
|
||||
BlobContinuationToken continuationToken = null;
|
||||
do
|
||||
{
|
||||
if (this.underLease)
|
||||
{
|
||||
await this.BlobManager.ConfirmLeaseIsGoodForAWhileAsync().ConfigureAwait(false);
|
||||
}
|
||||
var response = await this.pageBlobDirectory.ListBlobsSegmentedAsync(useFlatBlobListing: false, blobListingDetails: BlobListingDetails.None, maxResults: 1000,
|
||||
currentToken: continuationToken, options: BlobManager.BlobRequestOptionsWithRetry, operationContext: null)
|
||||
.ConfigureAwait(BlobManager.CONFIGURE_AWAIT_FOR_STORAGE_CALLS);
|
||||
|
||||
foreach (IListBlobItem item in response.Results)
|
||||
{
|
||||
if (item is CloudPageBlob pageBlob)
|
||||
{
|
||||
if (Int32.TryParse(pageBlob.Name.Replace(prefix, ""), out int segmentId))
|
||||
{
|
||||
this.BlobManager?.StorageTracer?.FasterStorageProgress($"AzureStorageDevice.StartAsync found segment={pageBlob.Name}");
|
||||
|
||||
bool ret = this.blobs.TryAdd(segmentId, new BlobEntry(pageBlob, this));
|
||||
|
||||
if (!ret)
|
||||
{
|
||||
throw new InvalidOperationException("Recovery of blobs is single-threaded and should not yield any failure due to concurrency");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
continuationToken = response.ContinuationToken;
|
||||
}
|
||||
while (continuationToken != null);
|
||||
|
||||
// find longest contiguous sequence at end
|
||||
var keys = this.blobs.Keys.ToList();
|
||||
if (keys.Count == 0)
|
||||
{
|
||||
// nothing has been written to this device so far.
|
||||
this.startSegment = 0;
|
||||
this.endSegment = -1;
|
||||
}
|
||||
else
|
||||
{
|
||||
keys.Sort();
|
||||
this.endSegment = keys.Last();
|
||||
for (int i = keys.Count - 2; i >= 0; i--)
|
||||
{
|
||||
if (keys[i] == keys[i + 1] - 1)
|
||||
{
|
||||
this.startSegment = i;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.BlobManager?.StorageTracer?.FasterStorageProgress($"AzureStorageDevice.StartAsync determined segment range for {this.pageBlobDirectory.Prefix}{this.blobName}: start={this.startSegment} end={this.endSegment}");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Is called on exceptions, if non-null; can be set by application
|
||||
/// </summary>
|
||||
internal BlobManager BlobManager { get; set; }
|
||||
|
||||
string GetSegmentBlobName(int segmentId)
|
||||
{
|
||||
return $"{this.blobName}.{segmentId}";
|
||||
}
|
||||
|
||||
//---- the overridden methods represent the interface for a generic storage device
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="StorageDeviceBase.Dispose">Inherited</see>
|
||||
/// </summary>
|
||||
public override void Dispose()
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="IDevice.RemoveSegmentAsync(int, AsyncCallback, IAsyncResult)"/>
|
||||
/// </summary>
|
||||
/// <param name="segment"></param>
|
||||
/// <param name="callback"></param>
|
||||
/// <param name="result"></param>
|
||||
public override void RemoveSegmentAsync(int segment, AsyncCallback callback, IAsyncResult result)
|
||||
{
|
||||
if (this.blobs.TryRemove(segment, out BlobEntry blob))
|
||||
{
|
||||
CloudPageBlob pageBlob = blob.PageBlob;
|
||||
|
||||
if (this.underLease)
|
||||
{
|
||||
this.BlobManager.ConfirmLeaseIsGoodForAWhile();
|
||||
}
|
||||
|
||||
if (!this.PartitionErrorHandler.IsTerminated)
|
||||
{
|
||||
pageBlob.DeleteAsync(cancellationToken: this.PartitionErrorHandler.Token)
|
||||
.ContinueWith((Task t) =>
|
||||
{
|
||||
if (t.IsFaulted)
|
||||
{
|
||||
this.BlobManager?.HandleBlobError(nameof(RemoveSegmentAsync), "could not remove page blob for segment", pageBlob?.Name, t.Exception, false, true);
|
||||
}
|
||||
callback(result);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="IDevice.ReadAsync(int, ulong, IntPtr, uint, DeviceIOCompletionCallback, object)">Inherited</see>
|
||||
/// </summary>
|
||||
public override unsafe void ReadAsync(int segmentId, ulong sourceAddress, IntPtr destinationAddress, uint readLength, DeviceIOCompletionCallback callback, object context)
|
||||
{
|
||||
this.BlobManager?.StorageTracer?.FasterStorageProgress($"AzureStorageDevice.ReadAsync Called segmentId={segmentId} sourceAddress={sourceAddress} readLength={readLength}");
|
||||
|
||||
// It is up to the allocator to make sure no reads are issued to segments before they are written
|
||||
if (!this.blobs.TryGetValue(segmentId, out BlobEntry blobEntry))
|
||||
{
|
||||
var nonLoadedBlob = this.pageBlobDirectory.GetPageBlobReference(this.GetSegmentBlobName(segmentId));
|
||||
var exception = new InvalidOperationException("Attempt to read a non-loaded segment");
|
||||
this.BlobManager?.HandleBlobError(nameof(ReadAsync), exception.Message, nonLoadedBlob?.Name, exception, true, false);
|
||||
throw exception;
|
||||
}
|
||||
|
||||
this.ReadFromBlobUnsafeAsync(blobEntry.PageBlob, (long)sourceAddress, (long)destinationAddress, readLength)
|
||||
.ContinueWith((Task t) =>
|
||||
{
|
||||
if (t.IsFaulted)
|
||||
{
|
||||
this.BlobManager?.StorageTracer?.FasterStorageProgress("AzureStorageDevice.ReadAsync Returned (Failure)");
|
||||
callback(uint.MaxValue, readLength, context);
|
||||
}
|
||||
else
|
||||
{
|
||||
this.BlobManager?.StorageTracer?.FasterStorageProgress("AzureStorageDevice.ReadAsync Returned");
|
||||
callback(0, readLength, context);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="IDevice.WriteAsync(IntPtr, int, ulong, uint, DeviceIOCompletionCallback, object)">Inherited</see>
|
||||
/// </summary>
|
||||
public override void WriteAsync(IntPtr sourceAddress, int segmentId, ulong destinationAddress, uint numBytesToWrite, DeviceIOCompletionCallback callback, object context)
|
||||
{
|
||||
this.BlobManager?.StorageTracer?.FasterStorageProgress($"AzureStorageDevice.WriteAsync Called segmentId={segmentId} destinationAddress={destinationAddress} numBytesToWrite={numBytesToWrite}");
|
||||
|
||||
if (!this.blobs.TryGetValue(segmentId, out BlobEntry blobEntry))
|
||||
{
|
||||
BlobEntry entry = new BlobEntry(this);
|
||||
if (this.blobs.TryAdd(segmentId, entry))
|
||||
{
|
||||
CloudPageBlob pageBlob = this.pageBlobDirectory.GetPageBlobReference(this.GetSegmentBlobName(segmentId));
|
||||
|
||||
// If segment size is -1 we use a default
|
||||
var size = this.segmentSize == -1 ? AzureStorageDevice.MAX_PAGEBLOB_SIZE : this.segmentSize;
|
||||
|
||||
// If no blob exists for the segment, we must first create the segment asynchronouly. (Create call takes ~70 ms by measurement)
|
||||
// After creation is done, we can call write.
|
||||
_ = entry.CreateAsync(size, pageBlob);
|
||||
}
|
||||
// Otherwise, some other thread beat us to it. Okay to use their blobs.
|
||||
blobEntry = this.blobs[segmentId];
|
||||
}
|
||||
this.TryWriteAsync(blobEntry, sourceAddress, destinationAddress, numBytesToWrite, callback, context);
|
||||
}
|
||||
|
||||
//---- The actual read and write accesses to the page blobs
|
||||
|
||||
unsafe Task WritePortionToBlobUnsafeAsync(CloudPageBlob blob, IntPtr sourceAddress, long destinationAddress, long offset, uint length)
|
||||
{
|
||||
return this.WritePortionToBlobAsync(new UnmanagedMemoryStream((byte*)sourceAddress + offset, length), blob, sourceAddress, destinationAddress, offset, length);
|
||||
}
|
||||
|
||||
async Task WritePortionToBlobAsync(UnmanagedMemoryStream stream, CloudPageBlob blob, IntPtr sourceAddress, long destinationAddress, long offset, uint length)
|
||||
{
|
||||
this.BlobManager?.StorageTracer?.FasterStorageProgress($"AzureStorageDevice.WritePortionToBlobAsync Called target={blob.Name} length={length} destinationAddress={destinationAddress + offset}");
|
||||
var stopwatch = new Stopwatch();
|
||||
|
||||
try
|
||||
{
|
||||
await BlobManager.AsynchronousStorageWriteMaxConcurrency.WaitAsync();
|
||||
|
||||
int numAttempts = 0;
|
||||
long streamPosition = stream.Position;
|
||||
|
||||
while(true) // retry loop
|
||||
{
|
||||
numAttempts++;
|
||||
try
|
||||
{
|
||||
if (this.underLease)
|
||||
{
|
||||
await this.BlobManager.ConfirmLeaseIsGoodForAWhileAsync().ConfigureAwait(false);
|
||||
}
|
||||
|
||||
this.BlobManager?.StorageTracer?.FasterStorageProgress($"starting upload target={blob.Name} length={length} destinationAddress={destinationAddress + offset} attempt={numAttempts}");
|
||||
stopwatch.Restart();
|
||||
|
||||
if (length > 0)
|
||||
{
|
||||
var blobRequestOptions = numAttempts > 2 ? BlobManager.BlobRequestOptionsDefault : BlobManager.BlobRequestOptionsAggressiveTimeout;
|
||||
|
||||
await blob.WritePagesAsync(stream, destinationAddress + offset,
|
||||
contentChecksum: null, accessCondition: null, options: blobRequestOptions, operationContext: null, cancellationToken: this.PartitionErrorHandler.Token)
|
||||
.ConfigureAwait(BlobManager.CONFIGURE_AWAIT_FOR_STORAGE_CALLS);
|
||||
}
|
||||
|
||||
stopwatch.Stop();
|
||||
|
||||
if (stopwatch.ElapsedMilliseconds > 1000)
|
||||
{
|
||||
this.BlobManager?.TraceHelper.FasterPerfWarning($"CloudPageBlob.WritePagesAsync took {stopwatch.ElapsedMilliseconds:f1}ms, which is excessive; target={blob.Name} length={length} destinationAddress={destinationAddress + offset}");
|
||||
}
|
||||
break;
|
||||
}
|
||||
catch (StorageException e) when (BlobUtils.IsTransientStorageError(e) && numAttempts < BlobManager.MaxRetries)
|
||||
{
|
||||
stopwatch.Stop();
|
||||
if (BlobUtils.IsTimeout(e))
|
||||
{
|
||||
this.BlobManager?.TraceHelper.FasterPerfWarning($"CloudPageBlob.WritePagesAsync timed out after {stopwatch.ElapsedMilliseconds:f1}ms, retrying now; numAttempts={numAttempts} target={blob.Name} length={length} destinationAddress={destinationAddress + offset}");
|
||||
}
|
||||
else
|
||||
{
|
||||
TimeSpan nextRetryIn = BlobManager.GetDelayBetweenRetries(numAttempts);
|
||||
this.BlobManager?.HandleBlobError(nameof(WritePortionToBlobAsync), $"could not write to page blob, will retry in {nextRetryIn}s, numAttempts={numAttempts}", blob?.Name, e, false, true);
|
||||
await Task.Delay(nextRetryIn);
|
||||
}
|
||||
stream.Seek(streamPosition, SeekOrigin.Begin); // must go back to original position before retry
|
||||
continue;
|
||||
}
|
||||
catch (Exception exception) when (!Utils.IsFatal(exception))
|
||||
{
|
||||
this.BlobManager?.HandleBlobError(nameof(WritePortionToBlobAsync), $"could not write to page blob target={blob.Name} length={length} destinationAddress={destinationAddress + offset}", blob?.Name, exception, true, this.PartitionErrorHandler.IsTerminated);
|
||||
throw;
|
||||
}
|
||||
};
|
||||
}
|
||||
finally
|
||||
{
|
||||
BlobManager.AsynchronousStorageWriteMaxConcurrency.Release();
|
||||
stream.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
unsafe Task ReadFromBlobUnsafeAsync(CloudPageBlob blob, long sourceAddress, long destinationAddress, uint readLength)
|
||||
{
|
||||
return this.ReadFromBlobAsync(new UnmanagedMemoryStream((byte*)destinationAddress, readLength, readLength, FileAccess.Write), blob, sourceAddress, destinationAddress, readLength);
|
||||
}
|
||||
|
||||
async Task ReadFromBlobAsync(UnmanagedMemoryStream stream, CloudPageBlob blob, long sourceAddress, long destinationAddress, uint readLength)
|
||||
{
|
||||
this.BlobManager?.StorageTracer?.FasterStorageProgress($"AzureStorageDevice.ReadFromBlobAsync Called target={blob.Name} readLength={readLength} sourceAddress={sourceAddress}");
|
||||
var stopwatch = new Stopwatch();
|
||||
|
||||
try
|
||||
{
|
||||
await BlobManager.AsynchronousStorageReadMaxConcurrency.WaitAsync();
|
||||
|
||||
int numAttempts = 0;
|
||||
|
||||
while (true) // retry loop
|
||||
{
|
||||
numAttempts++;
|
||||
try
|
||||
{
|
||||
if (this.underLease)
|
||||
{
|
||||
await this.BlobManager.ConfirmLeaseIsGoodForAWhileAsync().ConfigureAwait(false);
|
||||
}
|
||||
|
||||
this.BlobManager?.StorageTracer?.FasterStorageProgress($"starting download target={blob.Name} readLength={readLength} sourceAddress={sourceAddress} attempt={numAttempts}");
|
||||
stopwatch.Restart();
|
||||
|
||||
if (readLength > 0)
|
||||
{
|
||||
var blobRequestOptions = (numAttempts > 1 || readLength > MAX_UPLOAD_SIZE)
|
||||
? BlobManager.BlobRequestOptionsDefault : BlobManager.BlobRequestOptionsAggressiveTimeout;
|
||||
|
||||
await blob
|
||||
.DownloadRangeToStreamAsync(stream, sourceAddress, readLength, accessCondition: null, options: blobRequestOptions, operationContext: null, cancellationToken: this.PartitionErrorHandler.Token)
|
||||
.ConfigureAwait(BlobManager.CONFIGURE_AWAIT_FOR_STORAGE_CALLS);
|
||||
}
|
||||
|
||||
stopwatch.Stop();
|
||||
this.BlobManager?.StorageTracer?.FasterStorageProgress($"finished download target={blob.Name} readLength={readLength} sourceAddress={sourceAddress} latencyMs={stopwatch.Elapsed.TotalMilliseconds:F1}");
|
||||
|
||||
if (stopwatch.ElapsedMilliseconds > 1000)
|
||||
{
|
||||
this.BlobManager?.TraceHelper.FasterPerfWarning($"CloudPageBlob.DownloadRangeToStreamAsync took {stopwatch.ElapsedMilliseconds / 1000}s, which is excessive; target={blob.Name} readLength={readLength} sourceAddress={sourceAddress}");
|
||||
}
|
||||
|
||||
if (stream.Position != readLength)
|
||||
{
|
||||
throw new InvalidDataException($"wrong amount of data received from page blob, expected={readLength}, actual={stream.Position}");
|
||||
}
|
||||
break;
|
||||
}
|
||||
catch (StorageException e) when (BlobUtils.IsTransientStorageError(e) && numAttempts < BlobManager.MaxRetries)
|
||||
{
|
||||
stopwatch.Stop();
|
||||
if (BlobUtils.IsTimeout(e))
|
||||
{
|
||||
this.BlobManager?.TraceHelper.FasterPerfWarning($"CloudPageBlob.DownloadRangeToStreamAsync timed out after {stopwatch.ElapsedMilliseconds:f1}ms, retrying now; numAttempts={numAttempts} target={blob.Name} readLength={readLength} sourceAddress={sourceAddress}");
|
||||
}
|
||||
else
|
||||
{
|
||||
TimeSpan nextRetryIn = BlobManager.GetDelayBetweenRetries(numAttempts);
|
||||
this.BlobManager?.HandleBlobError(nameof(ReadFromBlobAsync), $"could not read from page blob, will retry in {nextRetryIn}s, numAttempts={numAttempts}", blob?.Name, e, false, true);
|
||||
await Task.Delay(nextRetryIn);
|
||||
}
|
||||
stream.Seek(0, SeekOrigin.Begin); // must go back to original position before retrying
|
||||
continue;
|
||||
}
|
||||
catch (Exception exception) when (!Utils.IsFatal(exception))
|
||||
{
|
||||
this.BlobManager?.HandleBlobError(nameof(ReadFromBlobAsync), "could not read from page blob", blob?.Name, exception, true, this.PartitionErrorHandler.IsTerminated);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
BlobManager.AsynchronousStorageReadMaxConcurrency.Release();
|
||||
stream.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
void TryWriteAsync(BlobEntry blobEntry, IntPtr sourceAddress, ulong destinationAddress, uint numBytesToWrite, DeviceIOCompletionCallback callback, object context)
|
||||
{
|
||||
// If pageBlob is null, it is being created. Attempt to queue the write for the creator to complete after it is done
|
||||
if (blobEntry.PageBlob == null
|
||||
&& blobEntry.TryQueueAction(p => this.WriteToBlobAsync(p, sourceAddress, destinationAddress, numBytesToWrite, callback, context)))
|
||||
{
|
||||
return;
|
||||
}
|
||||
// Otherwise, invoke directly.
|
||||
this.WriteToBlobAsync(blobEntry.PageBlob, sourceAddress, destinationAddress, numBytesToWrite, callback, context);
|
||||
}
|
||||
|
||||
unsafe void WriteToBlobAsync(CloudPageBlob blob, IntPtr sourceAddress, ulong destinationAddress, uint numBytesToWrite, DeviceIOCompletionCallback callback, object context)
|
||||
{
|
||||
this.WriteToBlobAsync(blob, sourceAddress, (long)destinationAddress, numBytesToWrite)
|
||||
.ContinueWith((Task t) =>
|
||||
{
|
||||
if (t.IsFaulted)
|
||||
{
|
||||
this.BlobManager?.StorageTracer?.FasterStorageProgress("AzureStorageDevice.WriteAsync Returned (Failure)");
|
||||
callback(uint.MaxValue, numBytesToWrite, context);
|
||||
}
|
||||
else
|
||||
{
|
||||
this.BlobManager?.StorageTracer?.FasterStorageProgress("AzureStorageDevice.WriteAsync Returned");
|
||||
callback(0, numBytesToWrite, context);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async Task WriteToBlobAsync(CloudPageBlob blob, IntPtr sourceAddress, long destinationAddress, uint numBytesToWrite)
|
||||
{
|
||||
long offset = 0;
|
||||
while (numBytesToWrite > 0)
|
||||
{
|
||||
var length = Math.Min(numBytesToWrite, MAX_UPLOAD_SIZE);
|
||||
await this.WritePortionToBlobUnsafeAsync(blob, sourceAddress, destinationAddress, offset, length).ConfigureAwait(false);
|
||||
numBytesToWrite -= length;
|
||||
offset += length;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Некоторые файлы не были показаны из-за слишком большого количества измененных файлов Показать больше
Загрузка…
Ссылка в новой задаче