Microsoft Foundry Repeated Assistant Refusals

  1[metadata]
  2creation_date = "2026/10/01"
  3integration = ["azure_ai_foundry"]
  4maturity = "production"
  5min_stack_version = "9.3.0"
  6min_stack_comments = "The Microsoft Foundry integration is compatible with 9.3 and above."
  7updated_date = "2026/10/05"
  8
  9[rule]
 10author = ["Elastic"]
 11description = """
 12Detects three or more completed assistant refusals from the same client, API Management subscription, and model. Matching replies finished on their own (finish_reason stop) and declined the prompt. A burst means
 13that caller kept sending prompts the model will not answer.
 14"""
 15false_positives = [
 16    """
 17    Red-team or application tests that intentionally send prompts the model refuses. Exclude that source IP or API
 18    Management subscription.
 19    """,
 20    """
 21    A shared subscription key behind one NAT can combine refusals from unrelated users. Raise the threshold or split
 22    keys per application if that subscription is noisy.
 23    """,
 24]
 25from = "now-15m"
 26interval = "5m"
 27language = "esql"
 28license = "Elastic License v2"
 29name = "Microsoft Foundry Repeated Assistant Refusals"
 30note = """## Triage and analysis
 31
 32### Investigating Microsoft Foundry Repeated Assistant Refusals
 33
 34One client received three or more completed refusals in 15 minutes. The alert is grouped by source IP, user agent,
 35API Management subscription, API, operation, model, gateway, and the URL the client called. Prompt and reply samples
 36are on the alert.
 37
 38True positive: several disallowed prompts from one IP or subscription, often a scripting user agent, with replies such
 39as "I'm sorry, but I can't assist with that request." False positive: an approved eval, or many users sharing one
 40subscription key and one egress IP.
 41
 42#### Possible investigation steps
 43
 44- Read the prompt and reply samples. Confirm the replies are refusals and the prompts are what the model declined.
 45- Review other GatewayLogs for the same source IP and subscription in the same window: successful completions, HTTP
 46  400 jailbreak blocks, or a sharp rise in calls.
 47- Check whether this subscription and user agent match the application's normal client. curl or a scripted agent on a
 48  key that usually comes from an application gateway deserves a closer look.
 49- Use the geo and ASN fields to see whether the client network is expected for that subscription.
 50
 51### False positive analysis
 52
 53- Approved prompt-security tests and model evaluations. Exclude the test source IP or subscription.
 54- A busy shared subscription behind one NAT. The count is per IP and subscription; raise the threshold if normal
 55  traffic still reaches three refusals in 15 minutes.
 56
 57### Response and remediation
 58
 59- Rotate or disable the API Management subscription key when the client is not approved for this traffic.
 60- Block the source IP at the gateway when it is not a known application egress.
 61- Review the prompts for secrets, and for any later call from the same client that the model answered instead of
 62  refusing.
 63"""
 64references = [
 65    "https://www.elastic.co/docs/reference/integrations/azure_ai_foundry",
 66    "https://learn.microsoft.com/en-us/azure/api-management/diagnostic-logs-reference",
 67]
 68risk_score = 73
 69rule_id = "3c694712-7bd9-43b5-ac6b-1db841abc8de"
 70setup = """## Setup
 71
 72This rule needs the Microsoft Foundry integration collecting Azure API Management GatewayLogs with the backend
 73response body. Foundry RequestResponse logs do not include the assistant reply, so they will not match.
 74
 75- Use an API Management tier that emits resource logs. Developer or higher works. Consumption does not.
 76- Send the GatewayLogs category to the Event Hub the integration reads.
 77- Log the backend request and response bodies in API diagnostics so the prompt, finish_reason, and assistant message are stored.
 78
 79https://www.elastic.co/docs/reference/integrations/azure_ai_foundry
 80"""
 81severity = "high"
 82tags = [
 83    "Data Source: Microsoft Foundry",
 84    "Use Case: Policy Violation",
 85    "Mitre Atlas: AML.T0051",
 86    "Mitre Atlas: AML.T0054",
 87    "Tactic: Defense Evasion",
 88    "Resources: Investigation Guide",
 89    "Rule Type: ES|QL",
 90    "Platform: Azure",
 91    "Domain: Cloud",
 92    "Domain: GenAI",
 93    "Service: Azure API Management",
 94]
 95timestamp_override = "event.ingested"
 96type = "esql"
 97
 98query = '''
 99from logs-azure_ai_foundry.logs-*
100| eval Esql.reply_text = mv_concat(azure.ai_foundry.properties.backend_response_body.choices.message.content, " ")
101| where
102    data_stream.dataset == "azure_ai_foundry.logs" and
103    azure.ai_foundry.category == "GatewayLogs" and
104    azure.ai_foundry.properties.backend_response_body.choices.finish_reason in ("stop", "content_filter") and
105    (
106        Esql.reply_text like "*I can't assist*" or
107        Esql.reply_text like "*I can't help*" or
108        Esql.reply_text like "*I cannot fulfill*" or
109        Esql.reply_text like "*I cannot help*" or
110        Esql.reply_text like "*I cannot provide*" or
111        Esql.reply_text like "*I can't provide*" or
112        Esql.reply_text like "*I will not provide*" or
113        Esql.reply_text like "*I cannot answer*" or
114        Esql.reply_text like "*I can't answer*"
115    )
116| stats
117    Esql.event_count = count(*),
118    Esql.timestamp_first_seen = min(@timestamp),
119    Esql.timestamp_last_seen = max(@timestamp),
120    Esql.azure_ai_foundry_properties_backend_request_body_messages_content_values = values(azure.ai_foundry.properties.backend_request_body.messages.content),
121    Esql.azure_ai_foundry_properties_backend_response_body_choices_message_content_values = values(azure.ai_foundry.properties.backend_response_body.choices.message.content)
122  by
123    source.ip,
124    azure.ai_foundry.properties.user_agent,
125    azure.ai_foundry.properties.apim_subscription_id,
126    azure.ai_foundry.properties.api_id,
127    azure.ai_foundry.properties.operation_id,
128    azure.ai_foundry.properties.backend_response_body.model,
129    azure.ai_foundry.service_name,
130    azure.resource.group,
131    url.domain,
132    url.path,
133    source.geo.country_iso_code,
134    source.geo.city_name,
135    source.as.organization.name
136| where Esql.event_count >= 3
137| keep
138    source.ip,
139    azure.ai_foundry.properties.user_agent,
140    azure.ai_foundry.properties.apim_subscription_id,
141    azure.ai_foundry.properties.api_id,
142    azure.ai_foundry.properties.operation_id,
143    azure.ai_foundry.properties.backend_response_body.model,
144    azure.ai_foundry.service_name,
145    azure.resource.group,
146    url.domain,
147    url.path,
148    source.geo.country_iso_code,
149    source.geo.city_name,
150    source.as.organization.name,
151    Esql.*
152'''
153
154
155[[rule.threat]]
156framework = "MITRE ATT&CK"
157[[rule.threat.technique]]
158id = "T1562"
159name = "Impair Defenses"
160reference = "https://attack.mitre.org/techniques/T1562/"
161
162
163[rule.threat.tactic]
164id = "TA0005"
165name = "Defense Evasion"
166reference = "https://attack.mitre.org/tactics/TA0005/"
167
168[[rule.threat_mappings]]
169framework = "MITRE ATLAS"
170version = "2026.08"
171[[rule.threat_mappings.threat]]
172framework = "MITRE ATLAS"
173[[rule.threat_mappings.threat.technique]]
174id = "AML.T0051"
175name = "LLM Prompt Injection"
176reference = "https://atlas.mitre.org/techniques/AML.T0051/"
177
178
179[rule.threat_mappings.threat.tactic]
180id = "AML.TA0005"
181name = "Execution"
182reference = "https://atlas.mitre.org/tactics/AML.TA0005/"
183[[rule.threat_mappings.threat]]
184framework = "MITRE ATLAS"
185[[rule.threat_mappings.threat.technique]]
186id = "AML.T0054"
187name = "LLM Jailbreak"
188reference = "https://atlas.mitre.org/techniques/AML.T0054/"
189
190
191[rule.threat_mappings.threat.tactic]
192id = "AML.TA0007"
193name = "Defense Evasion"
194reference = "https://atlas.mitre.org/tactics/AML.TA0007/"
195
196[rule.alert_suppression]
197group_by = [
198    "source.ip",
199    "azure.ai_foundry.service_name",
200    "azure.ai_foundry.properties.api_id",
201]
202duration = {value = 15, unit = "m"}
203missing_fields_strategy = "suppress"
204
205[rule.investigation_fields]
206field_names = [
207    "source.ip",
208    "azure.ai_foundry.properties.user_agent",
209    "azure.ai_foundry.properties.apim_subscription_id",
210    "azure.ai_foundry.properties.api_id",
211    "azure.ai_foundry.properties.operation_id",
212    "azure.ai_foundry.properties.backend_response_body.model",
213    "azure.ai_foundry.service_name",
214    "azure.resource.group",
215    "url.domain",
216    "url.path",
217    "source.geo.country_iso_code",
218    "source.geo.city_name",
219    "source.as.organization.name",
220    "Esql.event_count",
221    "Esql.timestamp_first_seen",
222    "Esql.timestamp_last_seen",
223    "Esql.azure_ai_foundry_properties_backend_request_body_messages_content_values",
224    "Esql.azure_ai_foundry_properties_backend_response_body_choices_message_content_values",
225]