Shell or Suspicious Process Spawned by Ray Worker

  1[metadata]
  2creation_date = "2026/10/29"
  3integration = ["endpoint"]
  4maturity = "production"
  5min_stack_comments = "MV_CONTAINS was added to ES|QL in 9.2.0 and is not available on 8.19; 9.3.0 is the lowest supported 9.x release."
  6min_stack_version = "9.4.0"
  7updated_date = "2026/11/03"
  8
  9[rule]
 10author = ["Elastic"]
 11description = """
 12Detects a Ray worker (`ray/_private/workers/default_worker.py`) spawning a shell, an inline Python
 13command (`python -c` / `python3 -c`), or a discovery or download utility. Ray runs job entrypoints from
 14that worker through a shell, so a Python reverse shell shows up as `sh -c` or `bash -c` whose command
 15line contains `python -c` and the program it starts, such as `/bin/sh -i`. On an exposed cluster this is
 16exploitation of the public-facing Ray jobs API (CVE-2023-48022). Ray commonly hosts LLM
 17training, fine-tuning, and serving, so this can be an attack on that infrastructure: theft of model
 18weights, datasets, or API and cloud tokens, tampering with a served model, or use of the GPUs for
 19cryptomining.
 20"""
 21from = "now-9m"
 22language = "esql"
 23license = "Elastic License v2"
 24name = "Shell or Suspicious Process Spawned by Ray Worker"
 25note = """## Triage and analysis
 26
 27### Investigating Shell or Suspicious Process Spawned by Ray Worker
 28
 29A Python process running `ray/_private/workers/default_worker.py` started `cmd`, PowerShell, a Unix shell,
 30inline `python -c` / `python3 -c`, `curl`, `wget`, `whoami`, `id`, `uname`, or `hostname`. Ray's job
 31supervisor is that worker. It runs the entrypoint with a shell, so the direct child is usually `sh -c` or
 32`cmd.exe /c` and the entrypoint text is on that command line. A `python3 -c` reverse shell that then runs
 33`/bin/sh -i` is visible on that shell command line. The `python3 -c` process and the interactive `/bin/sh`
 34are children of the shell, not of the worker, so they are separate events.
 35
 36### Possible investigation steps
 37
 38- Review `process.command_line` for the entrypoint. Downloaders, `whoami`, `id`, encoded commands, and
 39  reverse shells are higher priority than a known training or serving command.
 40- Confirm `process.parent.command_line` contains `ray/_private/workers/default_worker.py`.
 41- On the same host, review the child and grandchild processes and any outbound connections from that tree.
 42- Check whether the Ray dashboard (port 8265 by default) is reachable from outside the cluster, and who
 43  submitted the job. The submitter is not the parent process.
 44- Ask the workload owner whether this entrypoint is an approved Ray job.
 45
 46### False positive analysis
 47
 48- Approved Ray jobs start their entrypoint from this same worker, usually through `cmd.exe /c` on Windows or
 49  `sh -c` on Linux. Expect this rule on clusters that run jobs.
 50- Tune with the entrypoint command line, the Ray working directory, the host, or the user. Do not exclude
 51  `python` or `default_worker.py` broadly, or the worker respawn noise returns and the entrypoint shell is missed.
 52
 53### Response and remediation
 54
 55- If the entrypoint is not an approved job, stop the child process and isolate the host if it made outbound
 56  connections or read credentials.
 57- Remove unapproved files dropped by the job and rotate credentials present in the Ray environment.
 58- Stop exposing the dashboard. Bind it to a trusted interface and require authentication in front of it.
 59"""
 60references = [
 61    "https://www.oligo.security/blog/shadowray-attack-ai-workloads-actively-exploited-in-the-wild",
 62    "https://nvd.nist.gov/vuln/detail/CVE-2023-48022",
 63    "https://docs.ray.io/en/latest/ray-security/index.html",
 64]
 65risk_score = 47
 66rule_id = "fc7b9873-a9ed-43af-a9a2-c37660c4a211"
 67setup = """## Setup
 68
 69This rule is designed for data generated by [Elastic Defend](https://www.elastic.co/security/endpoint-security), which provides native endpoint detection and response, along with event enrichments designed to work with our detection rules.
 70
 71Setup instructions: https://ela.st/install-elastic-defend
 72"""
 73severity = "medium"
 74tags = [
 75    "Domain: Endpoint",
 76    "OS: Windows",
 77    "OS: Linux",
 78    "OS: macOS",
 79    "Use Case: Threat Detection",
 80    "Tactic: Initial Access",
 81    "Tactic: Execution",
 82    "Tactic: Discovery",
 83    "Data Source: Elastic Defend",
 84    "Resources: Investigation Guide",
 85    "Noise: High",
 86    "Threat: Vulnerability Exploit",
 87    "Performance: Normal",
 88    "Rule Type: ES|QL",
 89    "Platform: Windows",
 90    "Platform: Linux",
 91    "Platform: macOS",
 92    "Vuln: CVE-2023-48022",
 93]
 94timestamp_override = "event.ingested"
 95type = "esql"
 96
 97query = '''
 98FROM logs-endpoint.events.process-* METADATA _id, _version, _index
 99| WHERE KQL("""event.category:process and event.type:start""")
100  AND TO_LOWER(process.parent.name) LIKE "python*"
101  AND (
102    TO_LOWER(process.parent.command_line) LIKE """*\\ray\\_private\\workers\\default_worker.py*"""
103    OR TO_LOWER(process.parent.command_line) LIKE "*/ray/_private/workers/default_worker.py*"
104  )
105  AND (
106    TO_LOWER(process.name) IN (
107      "cmd.exe", "powershell.exe", "pwsh.exe",
108      "bash", "bash.exe", "sh", "zsh", "dash",
109      "curl", "curl.exe", "wget", "wget.exe",
110      "whoami", "whoami.exe", "id", "uname", "hostname", "hostname.exe"
111    )
112    // Direct python -c child of the worker.
113    OR (
114      TO_LOWER(process.name) LIKE "python*"
115      AND MV_CONTAINS(process.args, "-c")
116    )
117  )
118| KEEP
119    _id,
120    _version,
121    _index,
122    data_stream.namespace,
123    @timestamp,
124    host.name,
125    host.id,
126    host.os.type,
127    agent.id,
128    user.name,
129    user.domain,
130    process.name,
131    process.executable,
132    process.command_line,
133    process.args,
134    process.parent.name,
135    process.parent.executable,
136    process.parent.command_line,
137    process.parent.entity_id
138'''
139
140[rule.investigation_fields]
141field_names = [
142    "host.name",
143	"user.name",
144    "process.parent.name",
145    "process.parent.executable",
146    "process.parent.command_line",
147    "process.name",
148    "process.executable",
149    "process.command_line",
150]
151
152[[rule.threat]]
153framework = "MITRE ATT&CK"
154
155[[rule.threat.technique]]
156id = "T1190"
157name = "Exploit Public-Facing Application"
158reference = "https://attack.mitre.org/techniques/T1190/"
159
160[rule.threat.tactic]
161id = "TA0001"
162name = "Initial Access"
163reference = "https://attack.mitre.org/tactics/TA0001/"
164
165[[rule.threat]]
166framework = "MITRE ATT&CK"
167
168[[rule.threat.technique]]
169id = "T1059"
170name = "Command and Scripting Interpreter"
171reference = "https://attack.mitre.org/techniques/T1059/"
172
173[[rule.threat.technique.subtechnique]]
174id = "T1059.001"
175name = "PowerShell"
176reference = "https://attack.mitre.org/techniques/T1059/001/"
177
178[[rule.threat.technique.subtechnique]]
179id = "T1059.003"
180name = "Windows Command Shell"
181reference = "https://attack.mitre.org/techniques/T1059/003/"
182
183[[rule.threat.technique.subtechnique]]
184id = "T1059.004"
185name = "Unix Shell"
186reference = "https://attack.mitre.org/techniques/T1059/004/"
187
188[[rule.threat.technique.subtechnique]]
189id = "T1059.006"
190name = "Python"
191reference = "https://attack.mitre.org/techniques/T1059/006/"
192
193[rule.threat.tactic]
194id = "TA0002"
195name = "Execution"
196reference = "https://attack.mitre.org/tactics/TA0002/"
197
198[[rule.threat]]
199framework = "MITRE ATT&CK"
200
201[[rule.threat.technique]]
202id = "T1033"
203name = "System Owner/User Discovery"
204reference = "https://attack.mitre.org/techniques/T1033/"
205
206[[rule.threat.technique]]
207id = "T1082"
208name = "System Information Discovery"
209reference = "https://attack.mitre.org/techniques/T1082/"
210
211[rule.threat.tactic]
212id = "TA0007"
213name = "Discovery"
214reference = "https://attack.mitre.org/tactics/TA0007/"
215
216[[rule.threat]]
217framework = "MITRE ATT&CK"
218
219[[rule.threat.technique]]
220id = "T1105"
221name = "Ingress Tool Transfer"
222reference = "https://attack.mitre.org/techniques/T1105/"
223
224[rule.threat.tactic]
225id = "TA0011"
226name = "Command and Control"
227reference = "https://attack.mitre.org/tactics/TA0011/"