[get_normalized_search_id(1)]
description = Normalize Splunk Search ID
args = search_id
definition = rex field=$search_id$ mode=sed "s/^'|'$//g" \
| rex field=$search_id$ "_(?<search_id_normalized1>\d+[._]\d+)_" \
| rex field=$search_id$ "(?<search_id_normalized2>\d+[._]\d+$)" \
| rex field=$search_id$ "(?<search_id_normalized3>^\d+[._]\d+)" \
| eval search_id_normalized=if(isnull(search_id_normalized1),search_id_normalized2,search_id_normalized1) \
| eval search_id_normalized=if(isnull(search_id_normalized),search_id_normalized3,search_id_normalized) \
| eval search_id_normalized=if(isnull(search_id_normalized),search_id,search_id_normalized)\
| rex field=search_id_normalized mode=sed "s/\./_/g"\
| rex field=search_id_normalized mode=sed "s/^\w+;.*;|^_ACCELERATE_DM_|^_ACCELERATE_|_ACCELERATE_$//g"\
| fields - search_id_normalized1,search_id_normalized2,search_id_normalized3

(index=_audit sourcetype=audittrail source="*audit.log*" action=search)
| rex field=_raw max_match=0 "sourcetype_count__(?<sourcetype_val>[^=]+)=(?<eventCount>\d+)"
| rex field=_raw ",\ssavedsearch_name=\"(?<savedsearch_name>[^\"]+?)\""
| rex field=_raw ",\ssearch_id='(?<search_id>[^',]+)"
| rex field=_raw ",\sapp=\"(?<app>[^\"]+)\""
| rex field=_raw ",\suser=(?<user>[^,]+)"
| rex field=_raw ",\sinfo=(?<info>[^,]+)"
| rex field=_raw ",\ssearch='(?<searchQuery>[\W\w\n]+?)'((,\sautojoin=)|(\])|(,\sis_federated_search=)|(,\sincomplete_bucket_maps=)|(,\s[^\s=]+=))"
| eval searchQuery = replace(searchQuery, "',\s[^\s=]+='[^']+$", "")

index=_audit sourcetype=audittrail action=search (info=completed OR (info=granted AND search=*))
| rex field=_raw max_match=0 "sourcetype_count__(?<sourcetype_val>[^=]+)=(?<eventCount>\d+)"
| rex field=_raw ",\ssavedsearch_name=\"(?<savedsearch_label>[^\"]*)\"((,\s(search_startup_time=))|(\])|(,\sis_proxied=)|(,\ssearch_type=))"
| rex field=_raw ",\ssearch_id='(?<search_id>[^',]+)"
| rex field=_raw ",\sapp=\"(?<app>[^\"]+)\""
| rex field=_raw ",\suser=(?<user>[^,]+)" 
| rex field=_raw ",\sinfo=(?<info>[^,]+)"
| rex field=_raw ",\ssearch='(?<searchQuery>[\W\w\n]+?)'((,\sautojoin=)|(\])|(,\sis_federated_search=)|(,\sincomplete_bucket_maps=)|(,\s[^\s=]+=))"
| eval searchQuery = replace(searchQuery, "',\s[^\s=]+='[^']+$", "")
| fields _time, search_id, savedsearch_label, searchQuery, user, host, app, provenance, info, search_et, search_lt, exec_time, search_startup_time, total_run_time, sourcetype_val, event_count, result_count, available_count, scan_count, drop_count, searched_buckets, eliminated_buckets, considered_events, total_slices, decompressed_slices, duration_command_search_rawdata, duration_command_search_index, fully_completed_search, has_error_warn, is_federated_search, is_prjob, is_flex_search, is_proxied, is_*
| eval savedsearch_label = if(len(savedsearch_label) < 1 OR !match(savedsearch_label, "\w") OR match(savedsearch_label, "^search\d{1,3}$"), null(), savedsearch_label) 
``` Remove newlines and explicit tabs within SPL to make output table shorter ```
| eval searchQuery = replace(searchQuery, "\n", " ") 
| eval searchQuery = replace(searchQuery, "\t", "    ") 
| eval user = if(len(user) < 1 OR !match(user, "\w"), null(), user) 
| eval env = if(match(host, "splunkcloud\.com"), "Cloud", "OnPrem") 
``` Concatenate multivalue before passing through latest function ```
| foreach * [| eval <<FIELD>> = if( mvcount('<<FIELD>>') > 1, mvjoin('<<FIELD>>', ":~:"), '<<FIELD>>' )]
| stats max(_time) as latestAuditTime, latest(*) AS * by search_id, env
``` Convert concatenated MV back to proper MV ```
| foreach * [| eval <<FIELD>> = split('<<FIELD>>', ":~:")]
| rename searchQuery AS search, savedsearch_label AS savedsearch_name
| eval sid = replace( search_id, "'", "" ) 

| rest /servicesNS/-/-/search/jobs splunk_server=*
   search="sid!=SummaryDirector*" search="sid!=rsa_*" search="sid!=rt_*"
| rename title AS searchQuery, sid AS search_id, label AS savedsearch_label, eai:acl.app AS appName, eai:acl.owner AS owner, eventFieldCount AS rawEventFieldsReturned
   ``` FILTER - Drop any search jobs you know you don't want - Defaults are any search/jobs query and either of the presentation collection queries ```
| search NOT searchQuery IN ("*/search/jobs*", "*search_log_events*", "*search_jobs_rest_collection*", "*search_jobs_search_log_data*")
   ``` FILTER - Drop any SIDs that have already been written to the summary location
   Subsearch passes compiled filter string: splunk_server=<host> AND search_id IN (...)```
| search NOT [search index=summary sourcetype=search_jobs_rest_collection earliest=-4h@h latest=+4h@h
   | rex field=_raw ", search_id=\"?(?<search_id>[^\",]+)\"?,"
   | rex field=_raw "orig_splunk_server=\"(?<orig_splunk_server>[^\"]+)"
   | stats values(search_id) AS search_id_list BY orig_splunk_server
   | eval "Combined Subsearch Filter for REST API Collection Job" = "( splunk_server=\"" . orig_splunk_server . "\" AND search_id IN (\"" . mvjoin(search_id_list, "\", \"") . "\") )"
   | rename "Combined Subsearch Filter for REST API Collection Job" AS search
   | table search]
   ``` FILTER - to search artifacts published within last 4h - Timerange here should be same as in the search NOT filter above/below. ```
| eval _time = strptime(published, "%Y-%m-%dT%H:%M:%S.%3N")
| where _time >= relative_time(now(), "-4h")
   ``` Output to summary index will not include original SPL, if you want the original SPL join on sourcetype=audittrail with search_id ```
| table _time, splunk_server, search_id, savedsearch_label, appName, owner, dispatchState, keywords, runDuration, searchEarliestTime, searchLatestTime, diskUsage, rawEventFieldsReturned, resultCount, eventCount, scanCount, dropCount, searchTotalEliminatedBucketsCount, searchTotalBucketsCount, performance.command.search.index.invocations, performance.command.search.kv.duration_secs, priority, provenance, dispatchAs, request.ui_dispatch_app, eventSearch, published
   ``` Handle all empty string false-null fields returned from API by explicitly setting to null if length is less than 1 ```
| foreach *
   [| eval <<FIELD>> = if( isnull('<<FIELD>>') OR len('<<FIELD>>') < 1, null(), '<<FIELD>>')]
   ``` Generate normalized SID to populate savedsearch_label value properly across search artifacts ```
| `get_normalized_search_id(search_id)`
   ``` Aggregate key fields across SIDs using normalized SID - Note: min_sid_length is used to identify primary search ```
| eventstats min(eval(len(search_id))) AS min_sid_length, values(savedsearch_label) AS savedsearch_label, values(keywords) AS all_keywords BY search_id_normalized
| eval isPrimarySID = if( len(search_id)=min_sid_length AND !match(search_id, "^(remote|subsearch)"), 1, 0 )
| fields - min_sid_length
   ``` FILTER - Do not write summary index event if the search is not DONE - wait until this point to ensure label is passed to all subsearch artifacts ```
| search dispatchState="DONE"
   ``` Clean/Modify data as needed in preparation to write to summary index ```
| eval endpoint = "/servicesNS/-/-/search/jobs"
| foreach searchEarliestTime, searchLatestTime, runDuration
   [| eval <<FIELD>> = round('<<FIELD>>', 0)]
| addinfo | fields - info_*time | rename info_sid AS population_sid
   ``` Write output from search/jobs base collection to Summary Index - use appendpipe to avoid writing summary events in ad-hoc search ```
| appendpipe
   [| where match(population_sid, "^(scheduler|_scheduler_)")
   | collect testmode=f addinfo=f index=summary sourcetype=search_jobs_rest_collection marker="search_jobs_rest_collection_job"
   | where false()]

| rest /servicesNS/-/-/search/jobs splunk_server=*
   search="sid!=SummaryDirector*" search="sid!=rsa_*" search="sid!=rt_*"
| rename title AS searchQuery, sid AS search_id, label AS savedsearch_label, eai:acl.app AS appName, eai:acl.owner AS owner, eventFieldCount AS rawEventFieldsReturned
   ``` FILTER - Drop any search jobs you know you don't want - Defaults are any search/jobs query and either of the presentation collection queries ```
| search NOT searchQuery IN ("*/search/jobs*", "*search_log_events*", "*search_jobs_rest_collection*", "*search_jobs_search_log_data*")
   ``` FILTER - Drop any SIDs that have already been written to the summary location
   Subsearch passes compiled filter string: splunk_server=<host> AND search_id IN (...)```
| search NOT [search index=summary sourcetype=search_jobs_rest_collection earliest=-4h@h latest=+4h@h
   | rex field=_raw ", search_id=\"?(?<search_id>[^\",]+)\"?," | rex field=_raw "orig_splunk_server=\"(?<orig_splunk_server>[^\"]+)"
   | stats values(search_id) AS search_id_list BY orig_splunk_server
   | eval "Combined Subsearch Filter for REST API Collection Job" = "( splunk_server=\"" . orig_splunk_server . "\" AND search_id IN (\"" . mvjoin(search_id_list, "\", \"") . "\") )"
   | rename "Combined Subsearch Filter for REST API Collection Job" AS search | table search]
   ``` FILTER - to search artifacts published within last 4h - Timerange here should be same as in the search NOT filter above/below. ```
| eval _time = strptime(published, "%Y-%m-%dT%H:%M:%S.%3N")
| where _time >= relative_time(now(), "-4h")
   ``` Output to summary index will not include original SPL, if you want the original SPL join on sourcetype=audittrail with search_id ```
| table _time, splunk_server, search_id, savedsearch_label, appName, owner, dispatchState, keywords, runDuration, searchEarliestTime, searchLatestTime, diskUsage, rawEventFieldsReturned, resultCount, eventCount, scanCount, dropCount, searchTotalEliminatedBucketsCount, searchTotalBucketsCount, performance.command.search.index.invocations, performance.command.search.kv.duration_secs, priority, provenance, dispatchAs, request.ui_dispatch_app, eventSearch, published
   ``` Handle all empty string false-null fields returned from API by explicitly setting to null if length is less than 1 ```
| foreach * [| eval <<FIELD>> = if( isnull('<<FIELD>>') OR len('<<FIELD>>') < 1, null(), '<<FIELD>>')]
   ``` Generate normalized SID to populate savedsearch_label value properly across search artifacts ```
| `get_normalized_search_id(search_id)`
   ``` Aggregate key fields across SIDs using normalized SID - Note: min_sid_length is used to identify primary search ```
| eventstats min(eval(len(search_id))) AS min_sid_length, values(savedsearch_label) AS savedsearch_label, values(keywords) AS all_keywords BY search_id_normalized
| eval isPrimarySID = if( len(search_id)=min_sid_length AND !match(search_id, "^(remote|subsearch)"), 1, 0 )
| fields - min_sid_length
   ``` FILTER - Do not write summary index event if the search is not DONE - wait until this point to ensure label is passed to all subsearch artifacts ```
| search dispatchState="DONE"
   ``` Clean/Modify data as needed in preparation to write to summary index ```
| eval endpoint = "/servicesNS/-/-/search/jobs"
| foreach searchEarliestTime, searchLatestTime, runDuration
   [| eval <<FIELD>> = round('<<FIELD>>', 0)]
| addinfo | fields - info_*time | rename info_sid AS population_sid
   ``` Write output from search/jobs base collection to Summary Index - use appendpipe to avoid writing summary events in ad-hoc search ```
| appendpipe
   [| where match(population_sid, "^(scheduler|_scheduler_)")
   | collect testmode=f addinfo=f index=summary sourcetype=search_jobs_rest_collection marker="search_jobs_rest_collection_job"
   | where false()]
   ``` END search/jobs collection - BEGIN search.log collection ```
   ``` FILTER - Only pass the primary SIDs ```
| search isPrimarySID = 1
   ``` search.log can be split into multiple files for larger searches - here we generate new rows for each search ID to ensure we capture these expanded log files if they exist.
       NOTE: This will generate many search message errors since most of these will not exist most of the time. ```
| eval search_log_counter = split("search.log, search.log.1, search.log.2", ", ")  | mvexpand search_log_counter
   ``` Set the REST url the map command will query by using the search ID and the generated counter reference ```
| eval search_job_rest_url = "/servicesNS/-/-/search/jobs/" . search_id . "/" . search_log_counter
   ``` Run a new search to collect search.log for each input row, specifying the splunk_server that returned data previously. Pass the input field values by eval'ing them within map ```
| map maxsearches=200000
   search="| rest \"$search_job_rest_url$\" splunk_server=\"$splunk_server$\" | eval search_id = \"$search_id$\" | eval endpoint = \".../$search_log_counter$\" | eval app = \"$appName$\" | eval owner = \"$owner$\" | eval savedsearch_label = \"$savedsearch_label$\"
   | rename value AS search_log_line"
   ``` Convert search.log text block into multivalue, then split into individual rows ```
| makemv search_log_line tokenizer="([^\n]+)"
| stats first(*) AS * BY search_id, search_log_line
    ``` FILTER - Only keep search.log lines that we care about ```
| search search_log_line IN ("*BatchSearch is initialized*", "*Search requires the following indexes*", "*IndexScopedsearch is called for index*")
   ``` Clean/Modify search.log data as needed in preparation to write to summary index.
   NOTE: search_log_events raw format follows schema: <original search.log line> ~~~META:, <meta fields from collection job> ```
| rename splunk_server AS log_src_server
| eval _raw = search_log_line . " ~~~META:"
   ``` Manually create the _raw for the summary event, concatenating the original search.log line and the other fields from collection job in csv key=value format
   NOTE: This ensures a consistent ordering of fields. Do not change the order of the foreach field references without updating the regular expressions in search_id extractions as well. ```
| foreach app, endpoint, search_id, log_src_server, owner, savedsearch_label
   [| eval _raw = if( isnotnull('<<FIELD>>'), _raw . ", <<FIELD>>=\"" . '<<FIELD>>' . "\"", _raw)]
   ``` Write output of search.log collection to Summary Index - use appendpipe to avoid writing summary events in ad-hoc search ```
| appendpipe
   [| addinfo | fields - info_*time | rename info_sid AS population_sid
   | where match(population_sid, "^(scheduler|_scheduler_)") | fields - population_sid
   | collect testmode=f addinfo=f index=summary sourcetype=search_jobs_search_log_events marker="search_jobs_rest_collection_search_log_data"
   | where false()]

index=summary sourcetype=search_jobs_rest_collection isPrimarySID=1
| rex field=_raw ", search_id=\"?(?<search_id>[^\",]+)\"?,"
| rex field=_raw ", savedsearch_label=\"?(?<savedsearch_label>[^\",]+)\"?,"
| rex field=all_keywords max_match=0 "index::(?<index_keywords>[^\s\n]+)"
| rex field=all_keywords max_match=0  "sourcetype::(?<sourcetype_keywords>[^\s\n]+)"
| rex field=all_keywords max_match=0  "source::(?<source_keywords>[^\s\n]+)"
| stats dc(search_id) AS search_count, values(*_keywords) AS *_keywords 
BY savedsearch_label

index=summary sourcetype=search_jobs_search_log_events
| rex field=_raw ", search_id=\"?(?<search_id>[^\",]+)\"?,"
| rex field=_raw ", savedsearch_label=\"?(?<savedsearch_label>[^\",]+)\"?,"
| rex field=_raw "^(?<orig_search_log_line>.+?)~~~META:?,\s"
| rex field=orig_search_log_line "IndexScopedSearch is called for index = (?<index_usage_reference>[^\s,]+)"
| rex field=orig_search_log_line "Search requires the following indexes\s*=\s*\"\[(?<index_usage_array_reference>[^\]]+)"
| rex field=orig_search_log_line "BatchSearch is initialized for indexes\s*=\s*\{(?<index_usage_tuple_reference>[^\}]+)"
| eval index_usage_array_reference = split(index_usage_array_reference, ",")
| eval index_usage_tuple_reference = split(index_usage_tuple_reference, ",")
| stats dc(search_id) AS search_count, values(index_usage_*) AS index_usage_* BY savedsearch_label

rex field=$field$ max_match=100 "index\s*=[\s\"]?(?<Index_Reference1>[a-z0-9-_*]+)[\s\"]" 
| rex field=$field$ max_match=100 "index\s*=\s*\"?(?<Index_Reference2>_[a-z]+)[\s\"]" 
| rex field=$field$ max_match=100 "index=(?<Index_Reference3>[a-z0-9-_*]+)" 
| rex field=$field$ max_match=100 "index=(?<Index_Reference4>[`a-z0-9-_*]+)" 
| rex field=$field$ max_match=100 "index=(?<Index_Reference5>\w+)"
| rex field=$field$ max_match=100 "\|\s*collect\s+(?<Index_Reference6>`\S+`)" 
| eval Index_Reference = mvdedup(trim(mvappend(Index_Reference1,Index_Reference2,Index_Reference3,Index_Reference4,Index_Reference5,Index_Reference6))) 
| eval Index_Reference = if(match($field$, "index\s*=\s*_\*"), "all-internal-indexes", Index_Reference) 
| eval Index_Reference = if(match($field$, "index\s*=\s*\*|index=\"\*\""), "all-indexes", Index_Reference) 
| eval Index_Reference = mvfilter((!match(Index_Reference,"^1$"))) 
| eval Index_Reference = if(isnull(Index_Reference) OR Index_Reference="", "no-index-reference", Index_Reference) 
| fields - Index_Reference1 Index_Reference2 Index_Reference3 Index_Reference4 Index_Reference5 Index_Reference6

[get_index_reference(1)]
description = Extracts Any Reference to Index(es) (Quick)
args = field
definition = rex field=$field$ max_match=100 "index\s*=[\s\"]?(?<Index_Reference1>[a-z0-9-_*]+)[\s\"]" \
| rex field=$field$ max_match=100 "index\s*=\s*\"?(?<Index_Reference2>_[a-z]+)[\s\"]" \
| rex field=$field$ max_match=100 "index=(?<Index_Reference3>[a-z0-9-_*]+)" \
| rex field=$field$ max_match=100 "index=(?<Index_Reference4>[`a-z0-9-_*]+)" \
| rex field=$field$ max_match=100 "index=(?<Index_Reference5>\w+)"\
| rex field=$field$ max_match=100 "\|\s*collect\s+(?<Index_Reference6>`\S+`)" \
| eval Index_Reference = mvdedup(trim(mvappend(Index_Reference1,Index_Reference2,Index_Reference3,Index_Reference4,Index_Reference5,Index_Reference6))) \
| eval Index_Reference = if(match($field$, "index\s*=\s*_\*"), "all-internal-indexes", Index_Reference) \
| eval Index_Reference = if(match($field$, "index\s*=\s*\*|index=\"\*\""), "all-indexes", Index_Reference) \
| eval Index_Reference = mvfilter((!match(Index_Reference,"^1$"))) \
| eval Index_Reference = if(isnull(Index_Reference) OR Index_Reference="", "no-index-reference", Index_Reference) \
| fields - Index_Reference1 Index_Reference2 Index_Reference3 Index_Reference4 Index_Reference5 Index_Reference6

rex field=$field$ max_match=100 "sourcetype\s*!?=\s*(?<Sourcetype_Reference>.*?)[\s]" 
| rex field=Sourcetype_Reference mode=sed "s/[\s\",=()|]//g" 
| eval Sourcetype_Reference = if(Sourcetype_Reference = "" OR match(Sourcetype_Reference, "\$") OR isnull(Sourcetype_Reference), "no-sourcetype-reference", Sourcetype_Reference) 
| eval Sourcetype_Reference = if(match($field$, "sourcetype\s*=\s*\*|sourcetype=\"\*\""), "all-sourcetypes", Sourcetype_Reference)

[get_sourcetype_reference(1)]
description = Extracts Any Reference to Source Type(s) (Quick)
args = field
definition = rex field=$field$ max_match=100 "sourcetype\s*!?=\s*(?<Sourcetype_Reference>.*?)[\s]" \
| rex field=Sourcetype_Reference mode=sed "s/[\s\",=()|]//g" \
| eval Sourcetype_Reference = if(Sourcetype_Reference = "" OR match(Sourcetype_Reference, "\$") OR isnull(Sourcetype_Reference), "no-sourcetype-reference", Sourcetype_Reference) \
| eval Sourcetype_Reference = if(match($field$, "sourcetype\s*=\s*\*|sourcetype=\"\*\""), "all-sourcetypes", Sourcetype_Reference)

rex field=$field$ max_match=100 "source\\s*=\\s*(?<Source_Reference1>.*?)[\\s\"\\|]" 
| rex field=Source_Reference1 mode=sed "s/^[\\s$?><()\\\\,^=]*//g" 
| rex field=$field$ max_match=100 "source\\s+IN\\s*\\((?<Source_Reference2>.*?)\\)" 
| makemv delim="," Source_Reference2 
| rex field=Source_Reference2 mode=sed "s/^[\\s$?><()\\\\,^=]*//g" 
| eval Source_Reference=coalesce(Source_Reference1,Source_Reference2), Source_Reference=mvfilter((! match(Source_Reference,"^source|^\"|^ifisnull|^if\(|\.\*|^Mvindex|^lower|^mvfilter|^mvsort|^spath|^trim"))), 
    Source_Reference=mvdedup(mvsort(Source_Reference)), Source_Reference=if(((Source_Reference == "") OR isnull(Source_Reference)),"no-source-reference",Source_Reference), Source_Reference=if(match($field$,"source\\s*=\\s*\\*|source=\"\\*\""),"all-sources", Source_Reference) 
| fields - Source_Reference1 Source_Reference2
| fillnull value="no-source-reference" Source_Reference

[get_source_reference(1)]
description = Extracts Any Reference to Source(s) (Quick)
args = field
definition = rex field=$field$ max_match=100 "source\\s*=\\s*(?<Source_Reference1>.*?)[\\s\"\\|]" \
| rex field=Source_Reference1 mode=sed "s/^[\\s$?><()\\\\,^=]*//g" \
| rex field=$field$ max_match=100 "source\\s+IN\\s*\\((?<Source_Reference2>.*?)\\)" \
| makemv delim="," Source_Reference2 \
| rex field=Source_Reference2 mode=sed "s/^[\\s$?><()\\\\,^=]*//g" \
| eval Source_Reference=coalesce(Source_Reference1,Source_Reference2), Source_Reference=mvfilter((! match(Source_Reference,"^source|^\"|^ifisnull|^if\(|\.\*|^Mvindex|^lower|^mvfilter|^mvsort|^spath|^trim"))), \
    Source_Reference=mvdedup(mvsort(Source_Reference)), Source_Reference=if(((Source_Reference == "") OR isnull(Source_Reference)),"no-source-reference",Source_Reference), Source_Reference=if(match($field$,"source\\s*=\\s*\\*|source=\"\\*\""),"all-sources", Source_Reference) \
| fields - Source_Reference1 Source_Reference2\
| fillnull value="no-source-reference" Source_Reference

rex field=$field$ max_match=100 "eventtype\\s*=\\s*(?<Eventtype_Reference1>.*?)[\\s\"\\|]" 
| rex field=Eventtype_Reference1 mode=sed "s/^[\\s$?><()\\\\,^=\\]\\[+]*//g" 
| rex field=$field$ max_match=100 "eventtype\\s+IN\\s*\\((?<Eventtype_Reference2>.*?)\\)" 
| makemv delim="," Eventtype_Reference2 
| rex field=Eventtype_Reference2 mode=sed "s/^[\\s$?><()\\\\,^=]*//g" 
| eval Eventtype_Reference=coalesce(Eventtype_Reference1,Eventtype_Reference2), Eventtype_Reference=mvfilter((! match(Eventtype_Reference,"^eventtype|^trim|ifisnull|^\""))), Eventtype_Reference=mvdedup(mvsort(Eventtype_Reference))
| eval Eventtype_Reference=if(((Eventtype_Reference == "") OR isnull(Eventtype_Reference)),"no-eventtype-reference",Eventtype_Reference)
| fields - Eventtype_Reference1 Eventtype_Reference2
| fillnull value="no-eventtype-reference" Eventtype_Reference

[get_eventtype_reference(1)]
description = Extracts Any Reference to Event Type(s) (Quick)
args = field
definition = rex field=$field$ max_match=100 "eventtype\\s*=\\s*(?<Eventtype_Reference1>.*?)[\\s\"\\|]" \
| rex field=Eventtype_Reference1 mode=sed "s/^[\\s$?><()\\\\,^=\\]\\[+]*//g" \
| rex field=$field$ max_match=100 "eventtype\\s+IN\\s*\\((?<Eventtype_Reference2>.*?)\\)" \
| makemv delim="," Eventtype_Reference2 \
| rex field=Eventtype_Reference2 mode=sed "s/^[\\s$?><()\\\\,^=]*//g" \
| eval Eventtype_Reference=coalesce(Eventtype_Reference1,Eventtype_Reference2), Eventtype_Reference=mvfilter((! match(Eventtype_Reference,"^eventtype|^trim|ifisnull|^\""))), Eventtype_Reference=mvdedup(mvsort(Eventtype_Reference))\
| eval Eventtype_Reference=if(((Eventtype_Reference == "") OR isnull(Eventtype_Reference)),"no-eventtype-reference",Eventtype_Reference)\
| fields - Eventtype_Reference1 Eventtype_Reference2\
| fillnull value="no-eventtype-reference" Eventtype_Reference

rex field=$field$ max_match=100 "`(?<Macro_Reference>\p{Any}+?)`" 
| rex field=Macro_Reference mode=sed "s/\"|\s+//g" 
| eval Macro_Reference = mvfilter((! match(Macro_Reference,"^\||^\)|^:|^\[|^comment|^ia4s_comment"))) 
| eval Macro_Reference = if(((Macro_Reference == "") OR isnull(Macro_Reference)), "no-macro-reference", Macro_Reference) 
| mvexpand Macro_Reference 
| rex field=Macro_Reference max_match=100 "(?<Macro_Name>^[a-zA-Z0-9_-]+)" 
| rex field=Macro_Reference max_match=100 "\((?<Macro_Args>.*?)\)" 
| makemv delim="," Macro_Args 
| eval Macro_Args_Count = mvcount(Macro_Args) 
| eval Macro_Title = if (isnull(Macro_Args_Count), Macro_Name, Macro_Name . "(" . Macro_Args_Count . ")") 
| eval Macro_Title = if(((Macro_Title == "") OR isnull(Macro_Title)), "no-macro-title", Macro_Title) 
| fields - Macro_Reference1 Macro_Name Macro_Args Macro_Args_Count

[get_macro_reference(1)]
description = Extracts Any Reference to Macros (Quick)
args = field
definition = rex field=$field$ max_match=100 "`(?<Macro_Reference>\p{Any}+?)`" \
| rex field=Macro_Reference mode=sed "s/\"|\s+//g" \
| eval Macro_Reference = mvfilter((! match(Macro_Reference,"^\||^\)|^:|^\[|^comment|^ia4s_comment"))) \
| eval Macro_Reference = if(((Macro_Reference == "") OR isnull(Macro_Reference)), "no-macro-reference", Macro_Reference) \
| mvexpand Macro_Reference \
| rex field=Macro_Reference max_match=100 "(?<Macro_Name>^[a-zA-Z0-9_-]+)" \
| rex field=Macro_Reference max_match=100 "\((?<Macro_Args>.*?)\)" \
| makemv delim="," Macro_Args \
| eval Macro_Args_Count = mvcount(Macro_Args) \
| eval Macro_Title = if (isnull(Macro_Args_Count), Macro_Name, Macro_Name . "(" . Macro_Args_Count . ")") \
| eval Macro_Title = if(((Macro_Title == "") OR isnull(Macro_Title)), "no-macro-title", Macro_Title) \
| fields - Macro_Reference1 Macro_Name Macro_Args Macro_Args_Count

rex field=$field$ max_match=100 "\|\s*inputlookup\s+(?<Input_Lookup>[^|]+)" 
| rex field=$field$ max_match=100 "\|\s*from\s+inputlookup:(?<From_Input_Lookup>[^|]+)" 
| rex field=$field$ max_match=100 "\|\s*from\s+lookup:(?<From_Lookup>[^|]+)" 
| rex field=$field$ max_match=100 "\|\s*outputlookup\s+(?<Output_Lookup>[^|]+)" 
| rex field=$field$ max_match=100 "\|\s*lookup\s+(?<Lookup_Lookup>[^|\s]+)" 
| eval Input_Lookup = "Input_Lookup:".Input_Lookup , From_Input_Lookup = "From_Input_Lookup:".From_Input_Lookup, From_Lookup = "From_Lookup:".From_Lookup, Output_Lookup = "Output_Lookup:".Output_Lookup, Lookup_Lookup = "Lookup_Lookup:".Lookup_Lookup
| eval Lookup_Reference=mvsort(mvdedup(lower(mvappend(Lookup_Lookup,Input_Lookup,From_Lookup,From_Input_Lookup,Output_Lookup)))) 
| rex field=Lookup_Reference mode=sed "s/\"|append=\w+|create_empty=\w+|createinapp=\w+|override_if_empty=\w+|event_time_field=\w+|output_format=\w+|local=\w+|update=\w+|key_field=\w+|enabled=\w+|max=\w+|type=\w+|\s+where\s+.*|\$//g" 
`ia4s_comment("| rex field=Lookup_Reference mode=sed "s/(\s|\]).*$//g" ")` 
| eval Lookup_Reference=if(((Lookup_Reference == "") OR isnull(Lookup_Reference)),"no-lookup-reference", mvsort(mvdedup(trim(Lookup_Reference)))) 
| fields - Input_Lookup,From_Input_Lookup,From_Lookup,Output_Lookup,Lookup_Lookup
| fillnull value="no-lookup-reference" Lookup_Reference

[get_lookup_reference(1)]
description = Extracts Any Reference to Lookups (Quick)
args = field
definition = rex field=$field$ max_match=100 "\|\s*inputlookup\s+(?<Input_Lookup>[^|]+)" \
| rex field=$field$ max_match=100 "\|\s*from\s+inputlookup:(?<From_Input_Lookup>[^|]+)" \
| rex field=$field$ max_match=100 "\|\s*from\s+lookup:(?<From_Lookup>[^|]+)" \
| rex field=$field$ max_match=100 "\|\s*outputlookup\s+(?<Output_Lookup>[^|]+)" \
| rex field=$field$ max_match=100 "\|\s*lookup\s+(?<Lookup_Lookup>[^|\s]+)" \
| eval Input_Lookup = "Input_Lookup:".Input_Lookup , From_Input_Lookup = "From_Input_Lookup:".From_Input_Lookup, From_Lookup = "From_Lookup:".From_Lookup, Output_Lookup = "Output_Lookup:".Output_Lookup, Lookup_Lookup = "Lookup_Lookup:".Lookup_Lookup\
| eval Lookup_Reference=mvsort(mvdedup(lower(mvappend(Lookup_Lookup,Input_Lookup,From_Lookup,From_Input_Lookup,Output_Lookup)))) \
| rex field=Lookup_Reference mode=sed "s/\"|append=\w+|create_empty=\w+|createinapp=\w+|override_if_empty=\w+|event_time_field=\w+|output_format=\w+|local=\w+|update=\w+|key_field=\w+|enabled=\w+|max=\w+|type=\w+|\s+where\s+.*|\$//g" \
`ia4s_comment("| rex field=Lookup_Reference mode=sed "s/(\s|\]).*$//g" ")` \
| eval Lookup_Reference=if(((Lookup_Reference == "") OR isnull(Lookup_Reference)),"no-lookup-reference", mvsort(mvdedup(trim(Lookup_Reference)))) \
| fields - Input_Lookup,From_Input_Lookup,From_Lookup,Output_Lookup,Lookup_Lookup\
| fillnull value="no-lookup-reference" Lookup_Reference

rex field=$field$ max_match=100 "[fF][rR][oO][mM]\s*[dD][aA][tT][aA][mM][oO][dD][eE][lL][:=](?<Datamodel_Reference1>.*?)\s" 
| rex field=$field$ max_match=100 "\|\s*(datamodel|datamodelsimple)\s+(?<Datamodel_Reference2>.*?)\s" 
| eval Datamodel_Reference=coalesce(Datamodel_Reference1,Datamodel_Reference2) 
| rex field=Datamodel_Reference mode=sed "s/\"//g" 
| eval Datamodel_Reference = mvfilter( ! match(Datamodel_Reference, "^\$|type=|^\|") )
| eval Datamodel_Reference=if(((Datamodel_Reference == "") OR isnull(Datamodel_Reference)),"no-datamodel-reference", mvdedup(mvsort(Datamodel_Reference)))
| fields - Datamodel_Reference1 Datamodel_Reference2
| fillnull value="no-datamodel-reference" Datamodel_Reference

[get_datamodel_reference(1)]
description = Extracts Any Reference to Data Models (Quick)
args = field
definition = rex field=$field$ max_match=100 "[fF][rR][oO][mM]\s*[dD][aA][tT][aA][mM][oO][dD][eE][lL][:=](?<Datamodel_Reference1>.*?)\s" \
| rex field=$field$ max_match=100 "\|\s*(datamodel|datamodelsimple)\s+(?<Datamodel_Reference2>.*?)\s" \
| eval Datamodel_Reference=coalesce(Datamodel_Reference1,Datamodel_Reference2) \
| rex field=Datamodel_Reference mode=sed "s/\"//g" \
| eval Datamodel_Reference = mvfilter( ! match(Datamodel_Reference, "^\$|type=|^\|") )\
| eval Datamodel_Reference=if(((Datamodel_Reference == "") OR isnull(Datamodel_Reference)),"no-datamodel-reference", mvdedup(mvsort(Datamodel_Reference)))\
| fields - Datamodel_Reference1 Datamodel_Reference2\
| fillnull value="no-datamodel-reference" Datamodel_Reference