Use snake_case() and associated tests.

bigquery_etl/util/common.py

@@ -0,0 +1,24 @@
+"""Generic utility functions."""
+import re
+
+
+# Search for all camelCase situations in reverse with arbitrary lookaheads.
+REV_WORD_BOUND_PAT = re.compile(
+    r"""
+    \b                                  # standard word boundary
+    |(?<=[a-z][A-Z])(?=\d*[A-Z])        # A7Aa -> A7|Aa boundary
+    |(?<=[a-z][A-Z])(?=\d*[a-z])        # a7Aa -> a7|Aa boundary
+    |(?<=[A-Z])(?=\d*[a-z])             # a7A -> a7|A boundary
+    """,
+    re.VERBOSE,
+)
+
+
+def snake_case(line: str) -> str:
+    """Convert a string into a snake_cased string."""
+    # replace non-alphanumeric characters with spaces in the reversed line
+    subbed = re.sub(r"[^\w]|_", " ", line[::-1])
+    # apply the regex on the reversed string
+    words = REV_WORD_BOUND_PAT.split(subbed)
+    # filter spaces between words and snake_case and reverse again
+    return "_".join([w.lower() for w in words if w.strip()])[::-1]

sql/telemetry_derived/clients_daily_scalar_aggregates_v1.sql.py

@@ -12,6 +12,7 @@ from time import sleep
 
 sys.path.append(str(Path(__file__).parent.parent.parent.resolve()))
 from bigquery_etl.format_sql.formatter import reformat
+from bigquery_etl.util.common import snake_case
 
 
 PROBE_INFO_SERVICE = (
@@ -447,7 +448,7 @@ def get_scalar_probes(scalar_type):
         data = json.loads(gzip.decompress(url.read()).decode())
         scalar_probes = set(
             [
-                x.replace("scalar/", "").replace(".", "_")
+                snake_case(x.replace("scalar/", ""))
                 for x in data.keys()
                 if x.startswith("scalar/")
             ]

tests/resources/casing/alphanum_3.csv

@@ -0,0 +1,27 @@
+AAA,aaa
+AAa,a_aa
+AA7,aa7
+AaA,aa_a
+Aaa,aaa
+Aa7,aa7
+A7A,a7a
+A7a,a7a
+A77,a77
+aAA,a_aa
+aAa,a_aa
+aA7,a_a7
+aaA,aa_a
+aaa,aaa
+aa7,aa7
+a7A,a7_a
+a7a,a7a
+a77,a77
+7AA,7aa
+7Aa,7aa
+7A7,7a7
+7aA,7a_a
+7aa,7aa
+7a7,7a7
+77A,77a
+77a,77a
+777,777

tests/resources/casing/mps-diff-integration.csv

@@ -0,0 +1,232 @@
+AvailablePageFile,available_page_file
+AvailablePhysicalMemory,available_physical_memory
+AvailableVirtualMemory,available_virtual_memory
+BuildID,build_id
+D2DEnabled,d2d_enabled
+DWriteEnabled,d_write_enabled
+GPUActive,gpu_active
+Headless,headless
+IsGarbageCollecting,is_garbage_collecting
+LowEndMachine,low_end_machine
+ProductID,product_id
+ProductName,product_name
+RAM,ram
+ReleaseChannel,release_channel
+SecondsSinceLastCrash,seconds_since_last_crash
+StartupCrash,startup_crash
+SystemMemoryUsePercentage,system_memory_use_percentage
+TotalPageFile,total_page_file
+TotalPhysicalMemory,total_physical_memory
+TotalVirtualMemory,total_virtual_memory
+Version,version
+acceptLanguages,accept_languages
+accessibilityServices,accessibility_services
+activeAddons,active_addons
+activeExperiment,active_experiment
+activeGMPlugins,active_gm_plugins
+activePlugins,active_plugins
+adHocTablesDir,ad_hoc_tables_dir
+additionalProperties,additional_properties
+addonCompatibilityCheckEnabled,addon_compatibility_check_enabled
+addonId,addon_id
+addonVersion,addon_version
+advancedLayers,advanced_layers
+allowAutoplay,allow_autoplay
+apiCall,api_call
+apiVersion,api_version
+appDisabled,app_disabled
+appLocales,app_locales
+appName,app_name
+appUpdateChannel,app_update_channel
+appVersion,app_version
+appleModelId,apple_model_id
+applicationId,application_id
+applicationName,application_name
+architecturesInBinary,architectures_in_binary
+autoDownload,auto_download
+availableLocales,available_locales
+baseAddress,base_address
+blocklistEnabled,blocklist_enabled
+buildId,build_id
+certSubject,cert_subject
+changedFiles,changed_files
+changesetID,changeset_id
+clientId,client_id
+closedTS,closed_ts
+connType,conn_type
+crashDate,crash_date
+createdDate,created_date
+createdTimestamp,created_timestamp
+creationDate,creation_date
+debugID,debug_id
+debugName,debug_name
+defaultBrowser,default_browser
+defaultSearch,default_search
+defaultSearchEngine,default_search_engine
+defaultSearchEngineData,default_search_engine_data
+description,description
+detectedUri,detected_uri
+detectedVersion,detected_version
+deviceID,device_id
+displayVersion,display_version
+distributionId,distribution_id
+distributionVersion,distribution_version
+distributorChannel,distributor_channel
+driver,driver
+driverDate,driver_date
+driverVersion,driver_version
+e10sCohort,e10s_cohort
+e10sEnabled,e10s_enabled
+ecosystemClientId,ecosystem_client_id
+effectiveContentProcessLevel,effective_content_process_level
+encryptedData,encrypted_data
+encryptionKeyId,encryption_key_id
+engagedTS,engaged_ts
+engagementType,engagement_type
+errorModules,error_modules
+eventId,event_id
+expiredTS,expired_ts
+fileSize,file_size
+fileVersion,file_version
+firstUseDate,first_use_date
+firstView,first_view
+flashUsage,flash_usage
+flowId,flow_id
+globalSettings,global_settings
+gpuProcess,gpu_process
+hasBinaryComponents,has_binary_components
+hasCrashEnvironment,has_crash_environment
+hasSync,has_sync
+hotfixVersion,hotfix_version
+installDay,install_day
+installYear,install_year
+ipc_channel_error,ipc_channel_error
+isDefaultBrowser,is_default_browser
+isStartup,is_startup
+isStubProfile,is_stub_profile
+isSystem,is_system
+isTablet,is_tablet
+isWow64,is_wow64
+kernelVersion,kernel_version
+keyedHistograms,keyed_histograms
+l2cacheKB,l2cache_kb
+l3cacheKB,l3cache_kb
+landingSystem,landing_system
+lastBuildId,last_build_id
+lastVersion,last_version
+launcherProcessState,launcher_process_state
+learnMoreTS,learn_more_ts
+loadDurationMS,load_duration_ms
+loadPath,load_path
+loaderName,loader_name
+lostEventsCount,lost_events_count
+memoryMB,memory_mb
+mimeTypes,mime_types
+moduleName,module_name
+moduleTrustFlags,module_trust_flags
+offeredTS,offered_ts
+osName,os_name
+osVersion,os_version
+packetVersion,packet_version
+pageId,page_id
+pageSpecific,page_specific
+partnerId,partner_id
+partnerNames,partner_names
+pingDiscardedForSize,ping_discarded_for_size
+pioneerAddonMetadata,pioneer_addon_metadata
+pioneerId,pioneer_id
+pioneerUtilsVersion,pioneer_utils_version
+placesBookmarksCount,places_bookmarks_count
+placesPagesCount,places_pages_count
+platformVersion,platform_version
+pocketId,pocket_id
+previousBuildId,previous_build_id
+previousChannel,previous_channel
+previousVersion,previous_version
+prioData,prio_data
+processStartTimestamp,process_start_timestamp
+processType,process_type
+processUptimeMS,process_uptime_ms
+profileCreationDate,profile_creation_date
+profileDate,profile_date
+profileSubsessionCounter,profile_subsession_counter
+promptResponse,prompt_response
+pseudoDisplay,pseudo_display
+pushDate,push_date
+refreshRate,refresh_rate
+regionalPrefsLocales,regional_prefs_locales
+rememberCheckbox,remember_checkbox
+requestedLocales,requested_locales
+resetDate,reset_date
+responseTime,response_time
+reviewSystemUsed,review_system_used
+runId,run_id
+schemaName,schema_name
+schemaVersion,schema_version
+screenHeight,screen_height
+screenWidth,screen_width
+searchCohort,search_cohort
+searchCounts,search_counts
+sendFailure,send_failure
+servicePackMajor,service_pack_major
+servicePackMinor,service_pack_minor
+sessionId,session_id
+sessionState,session_state
+settingsChanged,settings_changed
+showTrackerStatsShare,show_tracker_stats_share
+signedState,signed_state
+sourcesJson,sources_json
+spbeMaxConcurrentTabCount,spbe_max_concurrent_tab_count
+spbeMaxConcurrentWindowCount,spbe_max_concurrent_window_count
+spbeNavigationAboutNewtab,spbe_navigation_about_newtab
+spbeNavigationContextmenu,spbe_navigation_contextmenu
+spbeNavigationSearchbar,spbe_navigation_searchbar
+spbeNavigationUrlbar,spbe_navigation_urlbar
+spbeTabOpenEventCount,spbe_tab_open_event_count
+spbeTotalUriCount,spbe_total_uri_count
+spbeUnfilteredUriCount,spbe_unfiltered_uri_count
+spbeUniqueDomainsCount,spbe_unique_domains_count
+spbeWindowOpenEventCount,spbe_window_open_event_count
+speedMHz,speed_m_hz
+sqlTableName,sql_table_name
+standardDeviation,standard_deviation
+structVersion,struct_version
+studyName,study_name
+submissionURL,submission_url
+subsessionId,subsession_id
+subsessionLength,subsession_length
+subsessionStartDate,subsession_start_date
+subsysID,subsys_id
+surveyId,survey_id
+surveyVersion,survey_version
+systemCpuCores,system_cpu_cores
+systemCpuSpeedMhz,system_cpu_speed_mhz
+systemGfxMonitors1ScreenWidth,system_gfx_monitors1_screen_width
+systemGfxMonitors1ScreenWidthZeroIndexed,system_gfx_monitors1_screen_width_zero_indexed
+systemLocales,system_locales
+systemMemoryMb,system_memory_mb
+tableName,table_name
+targetBuildId,target_build_id
+targetChannel,target_channel
+targetDisplayVersion,target_display_version
+targetVersion,target_version
+telemetryEnabled,telemetry_enabled
+textureSharing,texture_sharing
+threadID,thread_id
+threadName,thread_name
+timezoneOffest,timezone_offest
+totalBlockedAudibleMedia,total_blocked_audible_media
+totalPages,total_pages
+totalPagesAM,total_pages_am
+totalTime,total_time
+updateDay,update_day
+updaterAvailable,updater_available
+userDisabled,user_disabled
+vendorID,vendor_id
+virtualMaxMB,virtual_max_mb
+votedTS,voted_ts
+windowClosedTS,window_closed_ts
+windowsBuildNumber,windows_build_number
+windowsUBR,windows_ubr
+xpcomAbi,xpcom_abi
+xulLoadDurationMS,xul_load_duration_ms

tests/resources/casing/word_4.csv

@@ -0,0 +1,256 @@
+AAAA,aaaa
+AAAa,aa_aa
+AAA7,aaa7
+AAA_,aaa
+AAaA,a_aa_a
+AAaa,a_aaa
+AAa7,a_aa7
+AAa_,a_aa
+AA7A,aa7a
+AA7a,aa7a
+AA77,aa77
+AA7_,aa7
+AA_A,aa_a
+AA_a,aa_a
+AA_7,aa_7
+AA__,aa
+AaAA,aa_aa
+AaAa,aa_aa
+AaA7,aa_a7
+AaA_,aa_a
+AaaA,aaa_a
+Aaaa,aaaa
+Aaa7,aaa7
+Aaa_,aaa
+Aa7A,aa7_a
+Aa7a,aa7a
+Aa77,aa77
+Aa7_,aa7
+Aa_A,aa_a
+Aa_a,aa_a
+Aa_7,aa_7
+Aa__,aa
+A7AA,a7aa
+A7Aa,a7_aa
+A7A7,a7a7
+A7A_,a7a
+A7aA,a7a_a
+A7aa,a7aa
+A7a7,a7a7
+A7a_,a7a
+A77A,a77a
+A77a,a77a
+A777,a777
+A77_,a77
+A7_A,a7_a
+A7_a,a7_a
+A7_7,a7_7
+A7__,a7
+A_AA,a_aa
+A_Aa,a_aa
+A_A7,a_a7
+A_A_,a_a
+A_aA,a_a_a
+A_aa,a_aa
+A_a7,a_a7
+A_a_,a_a
+A_7A,a_7a
+A_7a,a_7a
+A_77,a_77
+A_7_,a_7
+A__A,a_a
+A__a,a_a
+A__7,a_7
+A___,a
+aAAA,a_aaa
+aAAa,a_a_aa
+aAA7,a_aa7
+aAA_,a_aa
+aAaA,a_aa_a
+aAaa,a_aaa
+aAa7,a_aa7
+aAa_,a_aa
+aA7A,a_a7a
+aA7a,a_a7a
+aA77,a_a77
+aA7_,a_a7
+aA_A,a_a_a
+aA_a,a_a_a
+aA_7,a_a_7
+aA__,a_a
+aaAA,aa_aa
+aaAa,aa_aa
+aaA7,aa_a7
+aaA_,aa_a
+aaaA,aaa_a
+aaaa,aaaa
+aaa7,aaa7
+aaa_,aaa
+aa7A,aa7_a
+aa7a,aa7a
+aa77,aa77
+aa7_,aa7
+aa_A,aa_a
+aa_a,aa_a
+aa_7,aa_7
+aa__,aa
+a7AA,a7_aa
+a7Aa,a7_aa
+a7A7,a7_a7
+a7A_,a7_a
+a7aA,a7a_a
+a7aa,a7aa
+a7a7,a7a7
+a7a_,a7a
+a77A,a77_a
+a77a,a77a
+a777,a777
+a77_,a77
+a7_A,a7_a
+a7_a,a7_a
+a7_7,a7_7
+a7__,a7
+a_AA,a_aa
+a_Aa,a_aa
+a_A7,a_a7
+a_A_,a_a
+a_aA,a_a_a
+a_aa,a_aa
+a_a7,a_a7
+a_a_,a_a
+a_7A,a_7a
+a_7a,a_7a
+a_77,a_77
+a_7_,a_7
+a__A,a_a
+a__a,a_a
+a__7,a_7
+a___,a
+7AAA,7aaa
+7AAa,7a_aa
+7AA7,7aa7
+7AA_,7aa
+7AaA,7aa_a
+7Aaa,7aaa
+7Aa7,7aa7
+7Aa_,7aa
+7A7A,7a7a
+7A7a,7a7a
+7A77,7a77
+7A7_,7a7
+7A_A,7a_a
+7A_a,7a_a
+7A_7,7a_7
+7A__,7a
+7aAA,7a_aa
+7aAa,7a_aa
+7aA7,7a_a7
+7aA_,7a_a
+7aaA,7aa_a
+7aaa,7aaa
+7aa7,7aa7
+7aa_,7aa
+7a7A,7a7_a
+7a7a,7a7a
+7a77,7a77
+7a7_,7a7
+7a_A,7a_a
+7a_a,7a_a
+7a_7,7a_7
+7a__,7a
+77AA,77aa
+77Aa,77aa
+77A7,77a7
+77A_,77a
+77aA,77a_a
+77aa,77aa
+77a7,77a7
+77a_,77a
+777A,777a
+777a,777a
+7777,7777
+777_,777
+77_A,77_a
+77_a,77_a
+77_7,77_7
+77__,77
+7_AA,7_aa
+7_Aa,7_aa
+7_A7,7_a7
+7_A_,7_a
+7_aA,7_a_a
+7_aa,7_aa
+7_a7,7_a7
+7_a_,7_a
+7_7A,7_7a
+7_7a,7_7a
+7_77,7_77
+7_7_,7_7
+7__A,7_a
+7__a,7_a
+7__7,7_7
+7___,7
+_AAA,aaa
+_AAa,a_aa
+_AA7,aa7
+_AA_,aa
+_AaA,aa_a
+_Aaa,aaa
+_Aa7,aa7
+_Aa_,aa
+_A7A,a7a
+_A7a,a7a
+_A77,a77
+_A7_,a7
+_A_A,a_a
+_A_a,a_a
+_A_7,a_7
+_A__,a
+_aAA,a_aa
+_aAa,a_aa
+_aA7,a_a7
+_aA_,a_a
+_aaA,aa_a
+_aaa,aaa
+_aa7,aa7
+_aa_,aa
+_a7A,a7_a
+_a7a,a7a
+_a77,a77
+_a7_,a7
+_a_A,a_a
+_a_a,a_a
+_a_7,a_7
+_a__,a
+_7AA,7aa
+_7Aa,7aa
+_7A7,7a7
+_7A_,7a
+_7aA,7a_a
+_7aa,7aa
+_7a7,7a7
+_7a_,7a
+_77A,77a
+_77a,77a
+_777,777
+_77_,77
+_7_A,7_a
+_7_a,7_a
+_7_7,7_7
+_7__,7
+__AA,aa
+__Aa,aa
+__A7,a7
+__A_,a
+__aA,a_a
+__aa,aa
+__a7,a7
+__a_,a
+__7A,7a
+__7a,7a
+__77,77
+__7_,7
+___A,a
+___a,a
+___7,7
+____,

tests/util/test_snake_casing.py

@@ -0,0 +1,29 @@
+import os
+import csv
+
+from pathlib import Path
+from bigquery_etl.util.common import snake_case
+
+
+def snake_case_test(case_name: str):
+    resource_path = Path("tests/resources/casing").resolve()
+    test_file = os.path.join(resource_path, case_name)
+
+    with open(test_file) as csv_file:
+        csv_reader = csv.reader(csv_file, delimiter=",")
+        for row in csv_reader:
+            assert len(row) == 2
+            assert snake_case(row[0]) == row[1]
+
+
+def test_snake_casing():
+    # all strings of length 3 drawn from the alphabet "aA7"
+    snake_case_test("alphanum_3.csv")
+
+    # all strings of length 4 drawn from the alphabet "aA7_"
+    snake_case_test("word_4.csv")
+
+    # all column names from mozilla-pipeline-schemas affected by snake_casing
+    # https://github.com/mozilla/jsonschema-transpiler/pull/79#issuecomment-509839572
+    # https://gist.github.com/acmiyaguchi/3f526c440b67ebe469bcb6ab2da5123f#file-readme-md
+    snake_case_test("mps-diff-integration.csv")

udf/main_summary_scalars.sql.py

@@ -3,18 +3,16 @@
 
 import itertools
 import json
-import re
+import sys
 import os.path
+from pathlib import Path
+
+sys.path.append(str(Path(__file__).parent.parent.resolve()))
+from bigquery_etl.util.common import snake_case  # noqa E402
 
 SCALAR_TYPES = {"uint": "INT64", "string": "STRING", "boolean": "BOOL"}
 
 
-def convert_camel_case(name):
-    """Convert camel case to snake case."""
-    s1 = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", name)
-    return re.sub("([a-z0-9])([A-Z])", r"\1_\2", s1).lower()
-
-
 def collect_probes(probes, schema_fields):
     """Collect scalars from probes and schema fields."""
     scalars = {"main": [], "content": [], "gpu": []}
@@ -39,7 +37,7 @@ def collect_probes(probes, schema_fields):
         for p in record_in_processes:
             collection[p].append(
                 (
-                    convert_camel_case(probe["name"]).replace(".", "_"),
+                    snake_case(probe["name"]).replace(".", "_"),
                     SCALAR_TYPES.get(
                         history[0]["details"]["kind"], history[0]["details"]["kind"]
                     ),