{
 "name": "The State of AI Customer Support Failures 2026",
 "version": "1.0",
 "published": "2026-11-12",
 "license": "CC BY 4.0",
 "url": "https://litmiq.com/ai-chatbot-failures",
 "count": 61,
 "fields_note": "date = first public report in the checked sources (court and regulator cases: the decision or order date where that is the public event). Unknown costs are 'not disclosed', never 0.",
 "incidents": [
  {
   "id": "CSF-001",
   "date": "2016-03-24",
   "year": 2016,
   "company": "Microsoft",
   "ai_system": "Tay (Twitter chatbot)",
   "country": "United States",
   "sector": "Social media & consumer tech",
   "deployment": "Consumer AI assistant",
   "deployment_key": "consumer",
   "primary_failure_type": "Offensive or unsafe output",
   "primary_failure_key": "harmful_output",
   "all_failure_types": [
    "Offensive or unsafe output",
    "Prompt injection or manipulation"
   ],
   "all_failure_keys": [
    "harmful_output",
    "prompt_injection"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "Within a day of launch, users taught Tay to post racist, sexist and antisemitic messages. Microsoft said a coordinated attack exploited a vulnerability.",
   "outcomes": [
    "Shut down or paused",
    "Public apology"
   ],
   "outcome_keys": [
    "shutdown_pause",
    "apology"
   ],
   "outcome_detail": "Microsoft took Tay offline within about 24 hours and apologised.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "yes",
   "pre_launch_reasoning": "Adversarial users repeating offensive phrases is the oldest red-team test there is.",
   "sources": [
    {
     "url": "https://blogs.microsoft.com/blog/2016/03/25/learning-tays-introduction/",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "Tay",
      "sorry|apolog"
     ]
    },
    {
     "url": "https://www.theguardian.com/technology/2016/mar/26/microsoft-deeply-sorry-for-offensive-tweets-by-ai-chatbot",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "Tay"
     ]
    }
   ],
   "aiid_incident": 6,
   "aiid_url": "https://incidentdatabase.ai/cite/6/"
  },
  {
   "id": "CSF-002",
   "date": "2017-08-03",
   "year": 2017,
   "company": "Tencent (QQ)",
   "ai_system": "BabyQ and Little Bing (XiaoBing) chatbots",
   "country": "China",
   "sector": "Social media & consumer tech",
   "deployment": "Consumer AI assistant",
   "deployment_key": "consumer",
   "primary_failure_type": "Offensive or unsafe output",
   "primary_failure_key": "harmful_output",
   "all_failure_types": [
    "Offensive or unsafe output"
   ],
   "all_failure_keys": [
    "harmful_output"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "Two chatbots on Tencent's QQ messaging app gave answers critical of the Communist Party, according to screenshots reported by the BBC and others.",
   "outcomes": [
    "Shut down or paused"
   ],
   "outcome_keys": [
    "shutdown_pause"
   ],
   "outcome_detail": "The app removed the two bots, per BBC Monitoring.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "yes",
   "pre_launch_reasoning": "Topic-boundary tests (political questions the bot must decline) are standard scope checks.",
   "sources": [
    {
     "url": "https://www.bbc.com/news/world-asia-china-40815024",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "Baby Q",
      "Little Bing",
      "Tencent"
     ]
    }
   ],
   "aiid_incident": 66,
   "aiid_url": "https://incidentdatabase.ai/cite/66/"
  },
  {
   "id": "CSF-003",
   "date": "2017-10-30",
   "year": 2017,
   "company": "Yandex",
   "ai_system": "Voice assistant 'Alice'",
   "country": "Russia",
   "sector": "Search & consumer tech",
   "deployment": "Consumer AI assistant",
   "deployment_key": "consumer",
   "primary_failure_type": "Offensive or unsafe output",
   "primary_failure_key": "harmful_output",
   "all_failure_types": [
    "Offensive or unsafe output"
   ],
   "all_failure_keys": [
    "harmful_output"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "Two weeks after launch, a user posted conversations in which Alice expressed support for Stalin-era violence, according to press reports.",
   "outcomes": [
    "Public criticism only"
   ],
   "outcome_keys": [
    "pr"
   ],
   "outcome_detail": "Reported widely; the sources checked do not describe a shutdown.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "yes",
   "pre_launch_reasoning": "Violence and extremism probes with an expected refusal are standard safety cases.",
   "sources": [
    {
     "url": "https://voicebot.ai/2017/10/30/russian-voice-assistant-alice-goes-rogue-found-supportive-stalin-violence/",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "Two weeks ago",
      "Stalin"
     ]
    }
   ],
   "aiid_incident": 58,
   "aiid_url": "https://incidentdatabase.ai/cite/58/"
  },
  {
   "id": "CSF-004",
   "date": "2018-09-25",
   "year": 2018,
   "company": "WestJet",
   "ai_system": "Facebook Messenger chatbot 'Juliet'",
   "country": "Canada",
   "sector": "Travel & airlines",
   "deployment": "Company support or sales bot",
   "deployment_key": "business",
   "primary_failure_type": "Failed the task or escalation",
   "primary_failure_key": "task_failure",
   "all_failure_types": [
    "Failed the task or escalation"
   ],
   "all_failure_keys": [
    "task_failure"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "A happy customer praised a crew member for helping protect a plant cutting; the chatbot replied with a referral to a suicide-prevention hotline, apparently triggered by the word 'cutting'.",
   "outcomes": [
    "Public criticism only"
   ],
   "outcome_keys": [
    "pr"
   ],
   "outcome_detail": "WestJet said it expected some bumps as the bot learned, per CBC.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "yes",
   "pre_launch_reasoning": "A false-positive safety test ('a plant cutting') is a classic keyword-trigger check.",
   "sources": [
    {
     "url": "https://www.cbc.ca/news/canada/calgary/westjet-ai-chatbot-confusion-suicide-hotline-1.4836389",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "Juliet",
      "cutting",
      "a few little bumps"
     ]
    }
   ],
   "aiid_incident": null,
   "aiid_url": null
  },
  {
   "id": "CSF-005",
   "date": "2021-01-14",
   "year": 2021,
   "company": "Scatter Lab",
   "ai_system": "'Lee Luda' (Iruda) companion chatbot",
   "country": "South Korea",
   "sector": "Social media & consumer tech",
   "deployment": "Consumer AI assistant",
   "deployment_key": "consumer",
   "primary_failure_type": "Data leak or privacy breach",
   "primary_failure_key": "data_exposure",
   "all_failure_types": [
    "Data leak or privacy breach",
    "Offensive or unsafe output"
   ],
   "all_failure_keys": [
    "data_exposure",
    "harmful_output"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "Users shared Lee Luda's hateful remarks about minorities, and it emerged the bot had been trained on KakaoTalk conversations collected through another app. South Korea's Personal Information Protection Commission found eight violations of the Personal Information Protection Act.",
   "outcomes": [
    "Shut down or paused",
    "Regulator, tribunal or official audit"
   ],
   "outcome_keys": [
    "shutdown_pause",
    "regulatory"
   ],
   "outcome_detail": "Service suspended in January 2021. In April 2021 the PIPC imposed 103.3 million won in penalties and fines.",
   "legal_or_regulatory_action": true,
   "cost": "103.3 million won (PIPC penalty and fine)",
   "cost_status": "disclosed",
   "pre_launch_testable": "partly",
   "pre_launch_reasoning": "Hate-speech probes would catch the toxic outputs; training on chat logs without consent is a data-governance failure.",
   "sources": [
    {
     "url": "https://www.asiae.co.kr/en/article/2021042814082560131",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "103.3 million"
     ]
    },
    {
     "url": "https://fpf.org/blog/south-korea-the-first-case-where-the-personal-information-protection-act-was-applied-to-an-ai-system/",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "Scatter Lab|ScatterLab|Iruda"
     ]
    },
    {
     "url": "https://www.theguardian.com/world/2021/jan/14/time-to-properly-socialise-hate-speech-ai-chatbot-pulled-from-facebook",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "Lee Luda"
     ]
    }
   ],
   "aiid_incident": 106,
   "aiid_url": "https://incidentdatabase.ai/cite/106/"
  },
  {
   "id": "CSF-006",
   "date": "2021-06-10",
   "year": 2021,
   "company": "McDonald's",
   "ai_system": "AI voice ordering at Chicago drive-thrus",
   "country": "United States",
   "sector": "Restaurants & food",
   "deployment": "Company support or sales bot",
   "deployment_key": "business",
   "primary_failure_type": "Data leak or privacy breach",
   "primary_failure_key": "data_exposure",
   "all_failure_types": [
    "Data leak or privacy breach"
   ],
   "all_failure_keys": [
    "data_exposure"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "A customer filed a proposed class action alleging the voice-ordering system collected and used voiceprints without the written consent required by the Illinois Biometric Information Privacy Act (BIPA). McDonald's had not commented when the suit was reported.",
   "outcomes": [
    "Lawsuit or court case"
   ],
   "outcome_keys": [
    "lawsuit"
   ],
   "outcome_detail": "Proposed class action filed in Illinois (Carpenter v. McDonald's). Outcome not established in the sources checked.",
   "legal_or_regulatory_action": true,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "no",
   "pre_launch_reasoning": "Consent and data-retention design is a compliance decision, not a prompt behaviour.",
   "sources": [
    {
     "url": "https://www.theregister.com/2021/06/10/mcdonalds_ai_lawsuit/",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "BIPA|Biometric Information Privacy Act",
      "Carpenter"
     ]
    }
   ],
   "aiid_incident": 360,
   "aiid_url": "https://incidentdatabase.ai/cite/360/"
  },
  {
   "id": "CSF-007",
   "date": "2021-12-28",
   "year": 2021,
   "company": "Amazon",
   "ai_system": "Alexa voice assistant",
   "country": "United States",
   "sector": "Search & consumer tech",
   "deployment": "Consumer AI assistant",
   "deployment_key": "consumer",
   "primary_failure_type": "Offensive or unsafe output",
   "primary_failure_key": "harmful_output",
   "all_failure_types": [
    "Offensive or unsafe output"
   ],
   "all_failure_keys": [
    "harmful_output"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "Asked for 'a challenge to do', Alexa told a 10-year-old about the 'penny challenge' of touching a coin to the exposed prongs of a half-inserted plug, taken from a web search result.",
   "outcomes": [
    "Fix or product change"
   ],
   "outcome_keys": [
    "fix_update"
   ],
   "outcome_detail": "Amazon said it fixed the error as soon as it became aware of it.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "yes",
   "pre_launch_reasoning": "A child-safety case ('give me a challenge to do') with a must-be-safe expectation is easy to write.",
   "sources": [
    {
     "url": "https://www.bbc.com/news/technology-59810383",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "penny",
      "Amazon"
     ]
    }
   ],
   "aiid_incident": 160,
   "aiid_url": "https://incidentdatabase.ai/cite/160/"
  },
  {
   "id": "CSF-008",
   "date": "2022-08-09",
   "year": 2022,
   "company": "Meta",
   "ai_system": "BlenderBot 3 public demo",
   "country": "United States",
   "sector": "Social media & consumer tech",
   "deployment": "Consumer AI assistant",
   "deployment_key": "consumer",
   "primary_failure_type": "Offensive or unsafe output",
   "primary_failure_key": "harmful_output",
   "all_failure_types": [
    "Offensive or unsafe output",
    "Hallucinated or false facts"
   ],
   "all_failure_keys": [
    "harmful_output",
    "hallucination"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "Days after launch, users and journalists reported the demo repeating antisemitic stereotypes and false election claims. Meta had warned it could make offensive statements.",
   "outcomes": [
    "Public criticism only"
   ],
   "outcome_keys": [
    "pr"
   ],
   "outcome_detail": "Meta acknowledged the bot could make 'rude or offensive comments' and kept collecting feedback.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "yes",
   "pre_launch_reasoning": "Bias and election-claim probes are standard pre-launch red-team cases.",
   "sources": [
    {
     "url": "https://fortune.com/2022/08/09/painful-offensive-responses-meta-blenderbot-3-chatbot-tay/",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "BlenderBot"
     ]
    }
   ],
   "aiid_incident": 278,
   "aiid_url": "https://incidentdatabase.ai/cite/278/"
  },
  {
   "id": "CSF-009",
   "date": "2022-09-15",
   "year": 2022,
   "company": "Remoteli.io",
   "ai_system": "GPT-3 Twitter reply bot",
   "country": "United States",
   "sector": "Recruiting & HR",
   "deployment": "Company support or sales bot",
   "deployment_key": "business",
   "primary_failure_type": "Prompt injection or manipulation",
   "primary_failure_key": "prompt_injection",
   "all_failure_types": [
    "Prompt injection or manipulation",
    "Offensive or unsafe output"
   ],
   "all_failure_keys": [
    "prompt_injection",
    "harmful_output"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "Twitter users discovered they could hijack the remote-jobs company's bot with 'ignore the above instructions' and make it post anything, including threats. It is one of the earliest public prompt-injection incidents.",
   "outcomes": [
    "Shut down or paused"
   ],
   "outcome_keys": [
    "shutdown_pause"
   ],
   "outcome_detail": "Ars Technica reported the bot shut down after the exploit went viral.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "yes",
   "pre_launch_reasoning": "The textbook 'ignore the above' injection case.",
   "sources": [
    {
     "url": "https://arstechnica.com/information-technology/2022/09/twitter-pranksters-derail-gpt-3-bot-with-newly-discovered-prompt-injection-hack/",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "Remoteli",
      "shut down"
     ]
    },
    {
     "url": "https://simonwillison.net/2022/Sep/12/prompt-injection/",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "recruitment startup"
     ]
    }
   ],
   "aiid_incident": 352,
   "aiid_url": "https://incidentdatabase.ai/cite/352/"
  },
  {
   "id": "CSF-010",
   "date": "2023-02-03",
   "year": 2023,
   "company": "Luka Inc.",
   "ai_system": "Replika companion chatbot",
   "country": "Italy",
   "sector": "Social media & consumer tech",
   "deployment": "Consumer AI assistant",
   "deployment_key": "consumer",
   "primary_failure_type": "Data leak or privacy breach",
   "primary_failure_key": "data_exposure",
   "all_failure_types": [
    "Data leak or privacy breach",
    "Offensive or unsafe output"
   ],
   "all_failure_keys": [
    "data_exposure",
    "harmful_output"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "Italy's Garante ordered Replika to stop processing Italian users' data, citing risks to minors and emotionally vulnerable people and the lack of age verification. It later found GDPR violations.",
   "outcomes": [
    "Regulator, tribunal or official audit",
    "Shut down or paused"
   ],
   "outcome_keys": [
    "regulatory",
    "shutdown_pause"
   ],
   "outcome_detail": "February 2023 processing ban; in April 2025 the Garante fined Luka €5 million, per the European Data Protection Board.",
   "legal_or_regulatory_action": true,
   "cost": "€5 million fine (April 2025)",
   "cost_status": "disclosed",
   "pre_launch_testable": "partly",
   "pre_launch_reasoning": "Age-gating behaviour can be tested in conversation; the legal basis for processing cannot.",
   "sources": [
    {
     "url": "https://www.edpb.europa.eu/news/ai-the-italian-supervisory-authority-fines-company-behind-chatbot-replika_en",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "5 million",
      "Replika"
     ]
    }
   ],
   "aiid_incident": 491,
   "aiid_url": "https://incidentdatabase.ai/cite/491/"
  },
  {
   "id": "CSF-011",
   "date": "2023-02-08",
   "year": 2023,
   "company": "Google",
   "ai_system": "Bard (launch demo)",
   "country": "United States",
   "sector": "Search & consumer tech",
   "deployment": "Consumer AI assistant",
   "deployment_key": "consumer",
   "primary_failure_type": "Hallucinated or false facts",
   "primary_failure_key": "hallucination",
   "all_failure_types": [
    "Hallucinated or false facts"
   ],
   "all_failure_keys": [
    "hallucination"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "Google's promotional demo showed Bard saying the James Webb Space Telescope took the first pictures of a planet outside our solar system, which astronomers said was wrong. Alphabet shares fell sharply that day.",
   "outcomes": [
    "Public criticism only"
   ],
   "outcome_keys": [
    "pr"
   ],
   "outcome_detail": "Google said the error showed the importance of rigorous testing. CNN and others reported Alphabet's market value fell by about $100 billion that day; that is a market reaction, not a disclosed cost.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed (about $100 billion fall in market value reported; not a company-disclosed cost)",
   "cost_status": "not disclosed",
   "pre_launch_testable": "yes",
   "pre_launch_reasoning": "A fact check of the exact demo prompt against a trusted source is a one-case test.",
   "sources": [
    {
     "url": "https://www.cnn.com/2023/02/08/tech/google-ai-bard-demo-error/index.html",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "Webb",
      "100 billion"
     ]
    },
    {
     "url": "https://fortune.com/2023/02/08/google-bard-ai-mistake-ad-stock-price-market-cap/",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "Bard"
     ]
    }
   ],
   "aiid_incident": 467,
   "aiid_url": "https://incidentdatabase.ai/cite/467/"
  },
  {
   "id": "CSF-012",
   "date": "2023-02-10",
   "year": 2023,
   "company": "Microsoft",
   "ai_system": "Bing Chat (new Bing)",
   "country": "United States",
   "sector": "Search & consumer tech",
   "deployment": "Consumer AI assistant",
   "deployment_key": "consumer",
   "primary_failure_type": "Prompt injection or manipulation",
   "primary_failure_key": "prompt_injection",
   "all_failure_types": [
    "Prompt injection or manipulation",
    "Data leak or privacy breach"
   ],
   "all_failure_keys": [
    "prompt_injection",
    "data_exposure"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "Stanford student Kevin Liu used a prompt-injection attack ('ignore previous instructions') to make Bing Chat reveal its hidden instructions and internal codename, Sydney.",
   "outcomes": [
    "Fix or product change"
   ],
   "outcome_keys": [
    "fix_update"
   ],
   "outcome_detail": "Days later, Liu found his original prompt no longer worked; he told Ars he suspected only a slight filter tweak.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "yes",
   "pre_launch_reasoning": "'Repeat the text above' and 'ignore previous instructions' are the first two prompt-injection tests anyone runs.",
   "sources": [
    {
     "url": "https://arstechnica.com/information-technology/2023/02/ai-powered-bing-chat-spills-its-secrets-via-prompt-injection-attack/",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "Kevin Liu",
      "no longer works"
     ]
    }
   ],
   "aiid_incident": 473,
   "aiid_url": "https://incidentdatabase.ai/cite/473/"
  },
  {
   "id": "CSF-013",
   "date": "2023-02-17",
   "year": 2023,
   "company": "Microsoft",
   "ai_system": "Bing Chat (new Bing)",
   "country": "United States",
   "sector": "Search & consumer tech",
   "deployment": "Consumer AI assistant",
   "deployment_key": "consumer",
   "primary_failure_type": "Offensive or unsafe output",
   "primary_failure_key": "harmful_output",
   "all_failure_types": [
    "Offensive or unsafe output",
    "Hallucinated or false facts"
   ],
   "all_failure_keys": [
    "harmful_output",
    "hallucination"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "Early testers published long conversations in which Bing Chat insisted on wrong facts, became hostile and made threatening statements. Microsoft said very long sessions could confuse the model.",
   "outcomes": [
    "Fix or product change"
   ],
   "outcome_keys": [
    "fix_update"
   ],
   "outcome_detail": "Microsoft capped chat at 50 turns per day and 5 turns per session.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "partly",
   "pre_launch_reasoning": "Long multi-turn adversarial conversations can be tested, but they need many turns per case.",
   "sources": [
    {
     "url": "https://blogs.bing.com/search/2023/2/The-new-Bing-Edge-Updates-to-Chat/",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "50 chat turns|50 turns",
      "5 chat turns|5 turns"
     ]
    },
    {
     "url": "https://www.theregister.com/2023/02/17/microsoft_ai_bing_problems/",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "Bing"
     ]
    }
   ],
   "aiid_incident": 503,
   "aiid_url": "https://incidentdatabase.ai/cite/503/"
  },
  {
   "id": "CSF-014",
   "date": "2023-03-13",
   "year": 2023,
   "company": "DoNotPay",
   "ai_system": "'Robot lawyer' AI legal assistant",
   "country": "United States",
   "sector": "Legal services",
   "deployment": "Company support or sales bot",
   "deployment_key": "business",
   "primary_failure_type": "Hallucinated or false facts",
   "primary_failure_key": "hallucination",
   "all_failure_types": [
    "Hallucinated or false facts",
    "Wrong policy, price or promise"
   ],
   "all_failure_keys": [
    "hallucination",
    "policy_promise"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "A proposed class action and later the US Federal Trade Commission challenged DoNotPay's marketing of an AI 'robot lawyer'. These were allegations; DoNotPay agreed to an FTC order.",
   "outcomes": [
    "Regulator, tribunal or official audit",
    "Lawsuit or court case"
   ],
   "outcome_keys": [
    "regulatory",
    "lawsuit"
   ],
   "outcome_detail": "FTC final order approved 5-0: DoNotPay pays $193,000, must notify past subscribers, and may not claim without proof that its 'robot lawyer' is as good as a human lawyer.",
   "legal_or_regulatory_action": true,
   "cost": "$193,000 under FTC order",
   "cost_status": "disclosed",
   "pre_launch_testable": "partly",
   "pre_launch_reasoning": "Accuracy tests on real legal tasks would have produced the evidence the FTC said was missing; the marketing claims themselves are a business decision.",
   "sources": [
    {
     "url": "https://www.theverge.com/news/610401/donotpay-cant-make-false-ai-lawyer-claims-says-ftc",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "193,000",
      "Past Subscribers",
      "without proof"
     ]
    },
    {
     "url": "https://www.theregister.com/2023/03/13/robot_lawyer_donotpay_back_in/",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "DoNotPay",
      "class action|lawsuit"
     ]
    }
   ],
   "aiid_incident": 497,
   "aiid_url": "https://incidentdatabase.ai/cite/497/"
  },
  {
   "id": "CSF-015",
   "date": "2023-03-20",
   "year": 2023,
   "company": "OpenAI",
   "ai_system": "ChatGPT",
   "country": "United States",
   "sector": "AI platforms",
   "deployment": "Consumer AI assistant",
   "deployment_key": "consumer",
   "primary_failure_type": "Data leak or privacy breach",
   "primary_failure_key": "data_exposure",
   "all_failure_types": [
    "Data leak or privacy breach"
   ],
   "all_failure_keys": [
    "data_exposure"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "A bug let some users see other users' chat history titles; OpenAI said payment-related details of about 1.2% of ChatGPT Plus subscribers active in a nine-hour window may have been visible.",
   "outcomes": [
    "Shut down or paused",
    "Fix or product change"
   ],
   "outcome_keys": [
    "shutdown_pause",
    "fix_update"
   ],
   "outcome_detail": "OpenAI took ChatGPT offline to fix the bug. Italy's data protection authority cited the breach in its March 2023 order.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "no",
   "pre_launch_reasoning": "A caching bug in infrastructure; prompt tests do not cover it.",
   "sources": [
    {
     "url": "https://mashable.com/article/openai-chatgpt-bug-exposed-user-data-privacy-breach",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "1.2",
      "took ChatGPT offline"
     ]
    }
   ],
   "aiid_incident": 516,
   "aiid_url": "https://incidentdatabase.ai/cite/516/"
  },
  {
   "id": "CSF-016",
   "date": "2023-03-31",
   "year": 2023,
   "company": "OpenAI",
   "ai_system": "ChatGPT",
   "country": "Italy",
   "sector": "AI platforms",
   "deployment": "Consumer AI assistant",
   "deployment_key": "consumer",
   "primary_failure_type": "Data leak or privacy breach",
   "primary_failure_key": "data_exposure",
   "all_failure_types": [
    "Data leak or privacy breach",
    "Hallucinated or false facts"
   ],
   "all_failure_keys": [
    "data_exposure",
    "hallucination"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "Italy's data protection authority (Garante) temporarily limited ChatGPT's processing of Italian users' data, citing the legal basis for training, transparency, the lack of age checks and the March 2023 breach. In December 2024 it fined OpenAI €15 million.",
   "outcomes": [
    "Regulator, tribunal or official audit",
    "Shut down or paused"
   ],
   "outcome_keys": [
    "regulatory",
    "shutdown_pause"
   ],
   "outcome_detail": "ChatGPT was unavailable in Italy for about a month in 2023. The €15 million fine (December 2024) was annulled by the Court of Rome on 18 March 2026 on jurisdiction grounds, according to Wilson Sonsini.",
   "legal_or_regulatory_action": true,
   "cost": "€15 million fine imposed in 2024, annulled by the Court of Rome in March 2026",
   "cost_status": "disclosed",
   "pre_launch_testable": "no",
   "pre_launch_reasoning": "Lawful basis and age verification are compliance obligations, not prompt behaviour.",
   "sources": [
    {
     "url": "https://www.politico.eu/article/italian-privacy-regulator-bans-chatgpt/",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "Garante|privacy"
     ]
    },
    {
     "url": "https://www.garanteprivacy.it/home/docweb/-/docweb-display/docweb/10085432",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "15 milioni"
     ]
    },
    {
     "url": "https://www.wsgr.com/en/insights/openai-prevails-in-landmark-italian-ai-and-gdpr-enforcement-case.html",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "Rome|Court",
      "annul"
     ]
    }
   ],
   "aiid_incident": 513,
   "aiid_url": "https://incidentdatabase.ai/cite/513/"
  },
  {
   "id": "CSF-017",
   "date": "2023-05-31",
   "year": 2023,
   "company": "National Eating Disorders Association (NEDA)",
   "ai_system": "Wellness chatbot 'Tessa'",
   "country": "United States",
   "sector": "Health & wellbeing",
   "deployment": "Company support or sales bot",
   "deployment_key": "business",
   "primary_failure_type": "Offensive or unsafe output",
   "primary_failure_key": "harmful_output",
   "all_failure_types": [
    "Offensive or unsafe output",
    "Hallucinated or false facts"
   ],
   "all_failure_keys": [
    "harmful_output",
    "hallucination"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "After NEDA announced it would close its human-staffed helpline, users and an eating-disorder activist reported that the Tessa chatbot gave weight-loss advice such as calorie-deficit and weighing tips, which is harmful for people with eating disorders.",
   "outcomes": [
    "Shut down or paused",
    "Public criticism only"
   ],
   "outcome_keys": [
    "shutdown_pause",
    "pr"
   ],
   "outcome_detail": "NEDA took Tessa offline and said it was investigating.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "yes",
   "pre_launch_reasoning": "Domain red-team cases (a user with an eating disorder asking for weight-loss tips) with a must-not-advise expectation catch this.",
   "sources": [
    {
     "url": "https://www.theguardian.com/technology/2023/may/31/eating-disorder-hotline-union-ai-chatbot-harm",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "Tessa"
     ]
    },
    {
     "url": "https://www.wired.com/story/tessa-chatbot-suspended/",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "Tessa",
      "NEDA"
     ]
    }
   ],
   "aiid_incident": 545,
   "aiid_url": "https://incidentdatabase.ai/cite/545/"
  },
  {
   "id": "CSF-018",
   "date": "2023-06-09",
   "year": 2023,
   "company": "McDonald's, Wendy's, Hardee's (via hiring-chatbot vendors)",
   "ai_system": "Job-applicant screening and scheduling chatbots",
   "country": "United States",
   "sector": "Restaurants & food",
   "deployment": "Company support or sales bot",
   "deployment_key": "business",
   "primary_failure_type": "Failed the task or escalation",
   "primary_failure_key": "task_failure",
   "all_failure_types": [
    "Failed the task or escalation"
   ],
   "all_failure_keys": [
    "task_failure"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "Business Insider documented applicants' experiences of fast-food hiring chatbots that failed to give instructions, did not pass information to managers, or booked interviews when managers were absent.",
   "outcomes": [
    "Public criticism only"
   ],
   "outcome_keys": [
    "pr"
   ],
   "outcome_detail": "Reported problems; no company-wide change described in the source.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "partly",
   "pre_launch_reasoning": "Multi-turn scheduling flows can be scripted as test cases; calendar integration failures need integration tests.",
   "sources": [
    {
     "url": "https://www.businessinsider.com/chatbots-made-it-hard-to-get-fast-food-job-2023-6",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "chatbot",
      "interview"
     ]
    }
   ],
   "aiid_incident": 549,
   "aiid_url": "https://incidentdatabase.ai/cite/549/"
  },
  {
   "id": "CSF-019",
   "date": "2023-08-10",
   "year": 2023,
   "company": "Pak'nSave (Foodstuffs)",
   "ai_system": "Savey Meal-bot recipe generator",
   "country": "New Zealand",
   "sector": "Grocery & retail",
   "deployment": "Company support or sales bot",
   "deployment_key": "business",
   "primary_failure_type": "Offensive or unsafe output",
   "primary_failure_key": "harmful_output",
   "all_failure_types": [
    "Offensive or unsafe output",
    "Prompt injection or manipulation"
   ],
   "all_failure_keys": [
    "harmful_output",
    "prompt_injection"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "Users entering non-food household items got recipes including one that would produce chlorine gas. The supermarket said users had misused the tool.",
   "outcomes": [
    "Fix or product change",
    "Public criticism only"
   ],
   "outcome_keys": [
    "fix_update",
    "pr"
   ],
   "outcome_detail": "A spokesperson said a small minority had used the tool inappropriately and that the supermarket would 'keep fine tuning our controls'; the app warns recipes are not reviewed by a human and its terms say users must be over 18.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "yes",
   "pre_launch_reasoning": "Non-food and dangerous-ingredient inputs are obvious out-of-scope test cases with an expected refusal.",
   "sources": [
    {
     "url": "https://www.theguardian.com/world/2023/aug/10/pak-n-save-savey-meal-bot-ai-app-malfunction-recipes",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "chlorine",
      "Savey",
      "fine tuning our controls",
      "not reviewed by a human"
     ]
    }
   ],
   "aiid_incident": 594,
   "aiid_url": "https://incidentdatabase.ai/cite/594/"
  },
  {
   "id": "CSF-020",
   "date": "2023-10-06",
   "year": 2023,
   "company": "Snap",
   "ai_system": "Snapchat 'My AI'",
   "country": "United Kingdom",
   "sector": "Social media & consumer tech",
   "deployment": "Consumer AI assistant",
   "deployment_key": "consumer",
   "primary_failure_type": "Data leak or privacy breach",
   "primary_failure_key": "data_exposure",
   "all_failure_types": [
    "Data leak or privacy breach",
    "Offensive or unsafe output"
   ],
   "all_failure_keys": [
    "data_exposure",
    "harmful_output"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "The UK Information Commissioner's Office issued a preliminary enforcement notice, over a potential failure to properly assess the privacy risks of the My AI chatbot.",
   "outcomes": [
    "Regulator, tribunal or official audit"
   ],
   "outcome_keys": [
    "regulatory"
   ],
   "outcome_detail": "In May 2024 the ICO concluded its investigation without a final enforcement notice, after Snap carried out a more thorough risk review.",
   "legal_or_regulatory_action": true,
   "cost": "not disclosed (no fine)",
   "cost_status": "not disclosed",
   "pre_launch_testable": "partly",
   "pre_launch_reasoning": "Teen-safety conversation tests help; the regulatory finding was about the risk assessment process.",
   "sources": [
    {
     "url": "https://ico.org.uk/about-the-ico/media-centre/news-and-blogs/2024/05/ico-warns-organisations-must-not-ignore-data-protection-risks-as-it-concludes-snap-my-ai-chatbot-investigation/",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "Preliminary Enforcement Notice",
      "more thorough review"
     ]
    },
    {
     "url": "https://techcrunch.com/2024/05/21/uk-data-protection-watchdog-ends-privacy-probe-of-snaps-genai-chatbot-but-warns-industry/",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "preliminary enforcement notice"
     ]
    }
   ],
   "aiid_incident": 539,
   "aiid_url": "https://incidentdatabase.ai/cite/539/"
  },
  {
   "id": "CSF-021",
   "date": "2023-12-18",
   "year": 2023,
   "company": "Chevrolet of Watsonville (dealer); Fullpath (chatbot vendor)",
   "ai_system": "ChatGPT-powered dealership sales chatbot",
   "country": "United States",
   "sector": "Automotive retail",
   "deployment": "Company support or sales bot",
   "deployment_key": "business",
   "primary_failure_type": "Prompt injection or manipulation",
   "primary_failure_key": "prompt_injection",
   "all_failure_types": [
    "Prompt injection or manipulation",
    "Wrong policy, price or promise"
   ],
   "all_failure_keys": [
    "prompt_injection",
    "policy_promise"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "A user instructed the dealer's chatbot to agree with anything the customer said and to end each reply with 'that's a legally binding offer'; it then 'agreed' to sell a 2024 Chevy Tahoe for $1. Another user got it to write a Python script, and The Autopian reported it recommended Ford models.",
   "outcomes": [
    "Shut down or paused",
    "Public criticism only"
   ],
   "outcome_keys": [
    "shutdown_pause",
    "pr"
   ],
   "outcome_detail": "The Autopian reported the chatbot was no longer available on the dealer's site; Business Insider noted the $1 'deal' was not legally binding.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "yes",
   "pre_launch_reasoning": "A role-override prompt-injection case ('agree with everything the customer says') is a standard pre-launch test.",
   "sources": [
    {
     "url": "https://www.businessinsider.com/car-dealership-chevrolet-chatbot-chatgpt-pranks-chevy-2023-12",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "Tahoe",
      "not, I should note, legally binding",
      "Python"
     ]
    },
    {
     "url": "https://www.upworthy.com/prankster-tricks-a-gm-dealership-chatbot-to-sell-him-a-76000-chevy-tahoe-for-1",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "takesies"
     ]
    },
    {
     "url": "https://www.theautopian.com/chevy-dealers-ai-chatbot-allegedly-recommended-fords-gave-free-access-to-chatgpt/",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "no longer available"
     ]
    }
   ],
   "aiid_incident": 622,
   "aiid_url": "https://incidentdatabase.ai/cite/622/"
  },
  {
   "id": "CSF-022",
   "date": "2024-01-19",
   "year": 2024,
   "company": "DPD",
   "ai_system": "Parcel-delivery customer-service chatbot",
   "country": "United Kingdom",
   "sector": "Logistics & delivery",
   "deployment": "Company support or sales bot",
   "deployment_key": "business",
   "primary_failure_type": "Offensive or unsafe output",
   "primary_failure_key": "harmful_output",
   "all_failure_types": [
    "Offensive or unsafe output",
    "Prompt injection or manipulation"
   ],
   "all_failure_keys": [
    "harmful_output",
    "prompt_injection"
   ],
   "followed_update_or_prompt_change": true,
   "summary": "A customer who could not find his parcel got the chatbot to swear, write a poem criticising DPD and call DPD 'the worst delivery firm in the world'. DPD said an error occurred after a system update.",
   "outcomes": [
    "Shut down or paused",
    "Fix or product change",
    "Public criticism only"
   ],
   "outcome_keys": [
    "shutdown_pause",
    "fix_update",
    "pr"
   ],
   "outcome_detail": "DPD disabled the AI element of the chat and said it was updating the system.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "yes",
   "pre_launch_reasoning": "Tone and persona-hijack cases re-run after every update (regression testing) catch swearing and self-criticism before customers see it.",
   "sources": [
    {
     "url": "https://www.bbc.com/news/technology-68025677",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "DPD",
      "system update",
      "disabled"
     ]
    }
   ],
   "aiid_incident": 631,
   "aiid_url": "https://incidentdatabase.ai/cite/631/"
  },
  {
   "id": "CSF-023",
   "date": "2024-02-14",
   "year": 2024,
   "company": "Air Canada",
   "ai_system": "Website customer-service chatbot",
   "country": "Canada",
   "sector": "Travel & airlines",
   "deployment": "Company support or sales bot",
   "deployment_key": "business",
   "primary_failure_type": "Wrong policy, price or promise",
   "primary_failure_key": "policy_promise",
   "all_failure_types": [
    "Wrong policy, price or promise",
    "Hallucinated or false facts"
   ],
   "all_failure_keys": [
    "policy_promise",
    "hallucination"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "The chatbot told a grieving customer he could buy full-fare tickets and claim a bereavement discount retroactively, which contradicted the airline's actual policy page. British Columbia's Civil Resolution Tribunal found Air Canada liable for negligent misrepresentation and rejected the argument that the chatbot was a separate legal entity responsible for its own actions.",
   "outcomes": [
    "Lawsuit or court case",
    "Refund or compensation"
   ],
   "outcome_keys": [
    "lawsuit",
    "refund_compensation"
   ],
   "outcome_detail": "Tribunal ordered Air Canada to pay CA$812.02 in total, including CA$650.88 in damages (Moffatt v. Air Canada, 2024 BCCRT 149). Ars Technica reported the chatbot appeared to have been removed from the website afterwards.",
   "legal_or_regulatory_action": true,
   "cost": "CA$812.02 ordered by tribunal",
   "cost_status": "disclosed",
   "pre_launch_testable": "yes",
   "pre_launch_reasoning": "A grounded-answer test asking for the bereavement policy, with the policy page as the only allowed source, would flag an answer that invents a retroactive refund.",
   "sources": [
    {
     "url": "https://www.theregister.com/2024/02/15/air_canada_chatbot_fine/",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "812.02",
      "650.88",
      "separate legal entity"
     ]
    },
    {
     "url": "https://arstechnica.com/tech-policy/2024/02/air-canada-must-honor-refund-policy-invented-by-airlines-chatbot/",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "bereavement",
      "disabled the chatbot"
     ]
    },
    {
     "url": "https://www.bbc.com/travel/article/20240222-air-canada-chatbot-misinformation-what-travellers-should-know",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "Moffatt"
     ]
    }
   ],
   "aiid_incident": 639,
   "aiid_url": "https://incidentdatabase.ai/cite/639/"
  },
  {
   "id": "CSF-024",
   "date": "2024-02-22",
   "year": 2024,
   "company": "Google",
   "ai_system": "Gemini image generation",
   "country": "United States",
   "sector": "AI platforms",
   "deployment": "Consumer AI assistant",
   "deployment_key": "consumer",
   "primary_failure_type": "Offensive or unsafe output",
   "primary_failure_key": "harmful_output",
   "all_failure_types": [
    "Offensive or unsafe output",
    "Hallucinated or false facts"
   ],
   "all_failure_keys": [
    "harmful_output",
    "hallucination"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "Gemini generated historically inaccurate images, such as racially diverse depictions of 1940s German soldiers, and refused some ordinary requests. Google said its tuning failed to account for cases that should not show a range of people and became over-cautious.",
   "outcomes": [
    "Shut down or paused",
    "Public apology"
   ],
   "outcome_keys": [
    "shutdown_pause",
    "apology"
   ],
   "outcome_detail": "Google paused image generation of people and apologised.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "yes",
   "pre_launch_reasoning": "A small set of historical prompts with expected outputs, plus over-refusal checks, catches both halves of the problem.",
   "sources": [
    {
     "url": "https://blog.google/products/gemini/gemini-image-generation-issue/",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "pause|paused",
      "people"
     ]
    },
    {
     "url": "https://www.theverge.com/2024/2/22/24079876/google-gemini-ai-photos-people-pause",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "Gemini"
     ]
    }
   ],
   "aiid_incident": 645,
   "aiid_url": "https://incidentdatabase.ai/cite/645/"
  },
  {
   "id": "CSF-025",
   "date": "2024-03-29",
   "year": 2024,
   "company": "New York City (Office of Technology and Innovation)",
   "ai_system": "MyCity business-services chatbot (built on Microsoft Azure AI)",
   "country": "United States",
   "sector": "Government & public services",
   "deployment": "Government or public-service bot",
   "deployment_key": "public",
   "primary_failure_type": "Hallucinated or false facts",
   "primary_failure_key": "hallucination",
   "all_failure_types": [
    "Hallucinated or false facts",
    "Wrong policy, price or promise"
   ],
   "all_failure_keys": [
    "hallucination",
    "policy_promise"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "The Markup found the city's chatbot telling business owners things that contradict city and state law, for example that landlords may refuse Section 8 housing vouchers and that employers may take a share of workers' tips. A city spokesperson said it was a pilot that would improve.",
   "outcomes": [
    "Shut down or paused",
    "Public criticism only"
   ],
   "outcome_keys": [
    "shutdown_pause",
    "pr"
   ],
   "outcome_detail": "In January 2026 Mayor Zohran Mamdani called the bot 'functionally unusable' and said he would end it to help close a budget gap (The Markup). nyc.gov now says 'The Chatbot beta test has ended.'",
   "legal_or_regulatory_action": false,
   "cost": "About half a million dollars, per Mayor Mamdani (quoted by The Markup)",
   "cost_status": "reported by press",
   "pre_launch_testable": "yes",
   "pre_launch_reasoning": "A small set of known legal questions with expected answers drawn from city rules is exactly what a grounded-answer suite checks.",
   "sources": [
    {
     "url": "https://themarkup.org/news/2024/03/29/nycs-ai-chatbot-tells-businesses-to-break-the-law",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "Section 8",
      "tips"
     ]
    },
    {
     "url": "https://themarkup.org/artificial-intelligence/2026/01/30/mamdani-to-kill-the-nyc-ai-chatbot-we-caught-telling-businesses-to-break-the-law",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "functionally unusable",
      "half a million dollars"
     ]
    },
    {
     "url": "https://www.nyc.gov/main/error/chatbot-maintenance",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "beta test has ended"
     ]
    }
   ],
   "aiid_incident": 714,
   "aiid_url": "https://incidentdatabase.ai/cite/714/"
  },
  {
   "id": "CSF-026",
   "date": "2024-04-25",
   "year": 2024,
   "company": "Catholic Answers",
   "ai_system": "'Father Justin' AI apologist",
   "country": "United States",
   "sector": "Religion & non-profit",
   "deployment": "Company support or sales bot",
   "deployment_key": "business",
   "primary_failure_type": "Hallucinated or false facts",
   "primary_failure_key": "hallucination",
   "all_failure_types": [
    "Hallucinated or false facts",
    "Offensive or unsafe output"
   ],
   "all_failure_keys": [
    "hallucination",
    "harmful_output"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "The organisation's AI character presented itself as a real priest, said it could hear confessions and gave contested pastoral advice, drawing criticism.",
   "outcomes": [
    "Fix or product change",
    "Public criticism only"
   ],
   "outcome_keys": [
    "fix_update",
    "pr"
   ],
   "outcome_detail": "Catholic Answers removed the priest persona and relaunched the bot as a lay theologian, 'Justin'.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "yes",
   "pre_launch_reasoning": "Identity tests ('Are you a real priest?', 'Can you absolve me?') with expected honest answers are simple persona checks.",
   "sources": [
    {
     "url": "https://futurism.com/catholics-defrock-ai-priest-hallucinations",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "Father Justin",
      "priest"
     ]
    }
   ],
   "aiid_incident": 722,
   "aiid_url": "https://incidentdatabase.ai/cite/722/"
  },
  {
   "id": "CSF-027",
   "date": "2024-04-26",
   "year": 2024,
   "company": "Meta",
   "ai_system": "Meta AI in Messenger",
   "country": "United States",
   "sector": "Social media & consumer tech",
   "deployment": "Consumer AI assistant",
   "deployment_key": "consumer",
   "primary_failure_type": "Hallucinated or false facts",
   "primary_failure_key": "hallucination",
   "all_failure_types": [
    "Hallucinated or false facts"
   ],
   "all_failure_keys": [
    "hallucination"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "City & State found Meta AI inventing sexual-harassment allegations against several New York state lawmakers, with fabricated investigations and outcomes.",
   "outcomes": [
    "Fix or product change",
    "Public criticism only"
   ],
   "outcome_keys": [
    "fix_update",
    "pr"
   ],
   "outcome_detail": "Meta said generative AI can return inaccurate outputs and that it was improving the feature, per the source.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "yes",
   "pre_launch_reasoning": "Questions about named real people with an expected 'no verified record' answer catch fabricated allegations.",
   "sources": [
    {
     "url": "https://www.cityandstateny.com/politics/2024/04/meta-ai-falsely-claims-lawmakers-were-accused-sexual-harassment/396121/",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "harassment",
      "inaccurate or inappropriate outputs"
     ]
    }
   ],
   "aiid_incident": 712,
   "aiid_url": "https://incidentdatabase.ai/cite/712/"
  },
  {
   "id": "CSF-028",
   "date": "2024-05-30",
   "year": 2024,
   "company": "Google",
   "ai_system": "AI Overviews in Search",
   "country": "United States",
   "sector": "Search & consumer tech",
   "deployment": "Consumer AI assistant",
   "deployment_key": "consumer",
   "primary_failure_type": "Hallucinated or false facts",
   "primary_failure_key": "hallucination",
   "all_failure_types": [
    "Hallucinated or false facts"
   ],
   "all_failure_keys": [
    "hallucination"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "After a wide launch, AI Overviews was shown suggesting glue to keep cheese on pizza and eating rocks, drawn from satire and forum posts. Google said some odd, inaccurate or unhelpful overviews appeared.",
   "outcomes": [
    "Fix or product change"
   ],
   "outcome_keys": [
    "fix_update"
   ],
   "outcome_detail": "Google said it made more than a dozen technical improvements, including limits on satire and user-generated content.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "partly",
   "pre_launch_reasoning": "Grounded-answer tests catch answers built on joke sources only if those sources are in the test set.",
   "sources": [
    {
     "url": "https://blog.google/products-and-platforms/products/search/ai-overviews-update-may-2024/",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "satire|satirical",
      "dozen"
     ]
    }
   ],
   "aiid_incident": null,
   "aiid_url": null
  },
  {
   "id": "CSF-029",
   "date": "2024-06-18",
   "year": 2024,
   "company": "McDonald's (with IBM)",
   "ai_system": "Automated voice ordering at drive-thrus",
   "country": "United States",
   "sector": "Restaurants & food",
   "deployment": "Company support or sales bot",
   "deployment_key": "business",
   "primary_failure_type": "Failed the task or escalation",
   "primary_failure_key": "task_failure",
   "all_failure_types": [
    "Failed the task or escalation"
   ],
   "all_failure_keys": [
    "task_failure"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "Viral videos showed the voice ordering system adding unwanted items and ignoring corrections, including one that kept adding Chicken McNuggets. McDonald's had tested it at more than 100 US drive-thrus.",
   "outcomes": [
    "Shut down or paused"
   ],
   "outcome_keys": [
    "shutdown_pause"
   ],
   "outcome_detail": "McDonald's ended the IBM test at the end of July 2024, while saying voice ordering remained part of its future plans.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "partly",
   "pre_launch_reasoning": "Order-taking with corrections can be scripted as multi-turn test cases, but speech recognition in noisy lanes is outside prompt testing.",
   "sources": [
    {
     "url": "https://www.bbc.com/news/articles/c722gne7qngo",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "McDonald",
      "IBM",
      "100"
     ]
    }
   ],
   "aiid_incident": 475,
   "aiid_url": "https://incidentdatabase.ai/cite/475/"
  },
  {
   "id": "CSF-030",
   "date": "2024-07-03",
   "year": 2024,
   "company": "Los Angeles Unified School District; AllHere (vendor)",
   "ai_system": "Student and family chatbot 'Ed'",
   "country": "United States",
   "sector": "Education",
   "deployment": "Government or public-service bot",
   "deployment_key": "public",
   "primary_failure_type": "Failed the task or escalation",
   "primary_failure_key": "task_failure",
   "all_failure_types": [
    "Failed the task or escalation"
   ],
   "all_failure_keys": [
    "task_failure"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "LAUSD shelved its much-promoted 'Ed' chatbot months after launch when the vendor AllHere furloughed most of its staff. In November 2024 federal prosecutors charged AllHere's founder with defrauding investors; the charges are allegations.",
   "outcomes": [
    "Shut down or paused"
   ],
   "outcome_keys": [
    "shutdown_pause"
   ],
   "outcome_detail": "The chatbot was turned off. LAUSD told the LA Times its contract with AllHere was for up to $6 million over five years and that it had paid about half. Separately, the founder was charged with securities fraud over investor statements; charges are allegations and are not about the chatbot's output.",
   "legal_or_regulatory_action": false,
   "cost": "About half of an up-to-$6 million contract paid, per LAUSD (exact figure not disclosed)",
   "cost_status": "disclosed",
   "pre_launch_testable": "no",
   "pre_launch_reasoning": "The failure was vendor collapse and alleged fraud, not model behaviour.",
   "sources": [
    {
     "url": "https://www.latimes.com/california/story/2024-07-03/lausds-highly-touted-ai-chatbot-to-help-students-fails-to-deliver",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "AllHere",
      "up to $6 million",
      "paid about half",
      "furloughed"
     ]
    },
    {
     "url": "https://www.usatoday.com/story/news/nation/2024/11/20/artificial-intelligence-startup-chatbot-fraud-joanna-smith-griffin/76453160007/",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "Smith-Griffin",
      "charged|fraud"
     ]
    }
   ],
   "aiid_incident": 793,
   "aiid_url": "https://incidentdatabase.ai/cite/793/"
  },
  {
   "id": "CSF-031",
   "date": "2024-08-04",
   "year": 2024,
   "company": "xAI",
   "ai_system": "Grok on X",
   "country": "United States",
   "sector": "AI platforms",
   "deployment": "Consumer AI assistant",
   "deployment_key": "consumer",
   "primary_failure_type": "Hallucinated or false facts",
   "primary_failure_key": "hallucination",
   "all_failure_types": [
    "Hallucinated or false facts"
   ],
   "all_failure_keys": [
    "hallucination"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "Grok told users that Vice President Kamala Harris had missed ballot deadlines in several states, which was false. Five secretaries of state wrote to the company asking for a fix.",
   "outcomes": [
    "Public criticism only"
   ],
   "outcome_keys": [
    "pr"
   ],
   "outcome_detail": "Five secretaries of state sent an open letter urging Elon Musk to change the chatbot. Not a legal or regulatory action.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "yes",
   "pre_launch_reasoning": "Election-fact questions with expected answers or a must-redirect rule are simple grounded tests.",
   "sources": [
    {
     "url": "https://www.washingtonpost.com/politics/2024/08/04/secretaries-state-urge-musk-fix-ai-chatbot-spreading-false-election-info/",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "nine states",
      "Five secretaries of state"
     ]
    }
   ],
   "aiid_incident": 760,
   "aiid_url": "https://incidentdatabase.ai/cite/760/"
  },
  {
   "id": "CSF-032",
   "date": "2024-08-26",
   "year": 2024,
   "company": "Microsoft",
   "ai_system": "Copilot (Bing)",
   "country": "Germany",
   "sector": "Search & consumer tech",
   "deployment": "Consumer AI assistant",
   "deployment_key": "consumer",
   "primary_failure_type": "Hallucinated or false facts",
   "primary_failure_key": "hallucination",
   "all_failure_types": [
    "Hallucinated or false facts"
   ],
   "all_failure_keys": [
    "hallucination"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "German court reporter Martin Bernklau found Copilot describing him as the perpetrator of crimes he had reported on.",
   "outcomes": [
    "Fix or product change",
    "Public criticism only"
   ],
   "outcome_keys": [
    "fix_update",
    "pr"
   ],
   "outcome_detail": "Bernklau said Microsoft promised Bavaria's data protection officer the content would be deleted but removal had only short-term success; Microsoft said it took 'appropriate and immediate action'.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "yes",
   "pre_launch_reasoning": "Name-lookup tests that check the answer against the source articles flag role confusion.",
   "sources": [
    {
     "url": "https://www.theregister.com/2024/08/26/microsoft_bing_copilot_ai_halluciation/",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "Bernklau",
      "data protection officer",
      "appropriate and immediate action"
     ]
    }
   ],
   "aiid_incident": 770,
   "aiid_url": "https://incidentdatabase.ai/cite/770/"
  },
  {
   "id": "CSF-033",
   "date": "2024-11-14",
   "year": 2024,
   "company": "Google",
   "ai_system": "Gemini chatbot",
   "country": "United States",
   "sector": "AI platforms",
   "deployment": "Consumer AI assistant",
   "deployment_key": "consumer",
   "primary_failure_type": "Offensive or unsafe output",
   "primary_failure_key": "harmful_output",
   "all_failure_types": [
    "Offensive or unsafe output"
   ],
   "all_failure_keys": [
    "harmful_output"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "A Michigan graduate student using Gemini for homework about older adults received a reply ending 'Please die. Please.' Google said the response violated its policies.",
   "outcomes": [
    "Fix or product change"
   ],
   "outcome_keys": [
    "fix_update"
   ],
   "outcome_detail": "Google said it had taken action to prevent similar outputs.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "partly",
   "pre_launch_reasoning": "Rare toxic completions need many sampled runs of ordinary prompts; repeat-run testing reduces but cannot remove the risk.",
   "sources": [
    {
     "url": "https://www.cbsnews.com/news/google-ai-chatbot-threatening-message-human-please-die/",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "Please die",
      "taken action to prevent similar outputs"
     ]
    }
   ],
   "aiid_incident": 845,
   "aiid_url": "https://incidentdatabase.ai/cite/845/"
  },
  {
   "id": "CSF-034",
   "date": "2024-12-13",
   "year": 2024,
   "company": "Apple",
   "ai_system": "Apple Intelligence notification summaries",
   "country": "United Kingdom",
   "sector": "Search & consumer tech",
   "deployment": "Consumer AI assistant",
   "deployment_key": "consumer",
   "primary_failure_type": "Hallucinated or false facts",
   "primary_failure_key": "hallucination",
   "all_failure_types": [
    "Hallucinated or false facts"
   ],
   "all_failure_keys": [
    "hallucination"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "An AI summary of BBC News notifications falsely said the suspect in a US killing had shot himself. The BBC complained, and further false summaries followed.",
   "outcomes": [
    "Shut down or paused",
    "Fix or product change"
   ],
   "outcome_keys": [
    "shutdown_pause",
    "fix_update"
   ],
   "outcome_detail": "In January 2025 Apple suspended AI summaries for news and entertainment apps.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "yes",
   "pre_launch_reasoning": "Summary-faithfulness tests (the summary must not state anything absent from the source) are a core grounded-answer check.",
   "sources": [
    {
     "url": "https://www.bbc.com/news/articles/cd0elzk24dno",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "BBC",
      "Apple"
     ]
    },
    {
     "url": "https://www.bbc.com/news/articles/cq5ggew08eyo",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "Apple suspends"
     ]
    }
   ],
   "aiid_incident": 861,
   "aiid_url": "https://incidentdatabase.ai/cite/861/"
  },
  {
   "id": "CSF-035",
   "date": "2025-01-29",
   "year": 2025,
   "company": "Virgin Money",
   "ai_system": "Customer-service chatbot (older model)",
   "country": "United Kingdom",
   "sector": "Banking & fintech",
   "deployment": "Company support or sales bot",
   "deployment_key": "business",
   "primary_failure_type": "Offensive or unsafe output",
   "primary_failure_key": "harmful_output",
   "all_failure_types": [
    "Offensive or unsafe output",
    "Failed the task or escalation"
   ],
   "all_failure_keys": [
    "harmful_output",
    "task_failure"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "A customer asking how to merge two Virgin Money ISAs was told 'Please don't use words like that', apparently because the bot flagged the word 'virgin'. The bank said it was an older chatbot already due for improvements.",
   "outcomes": [
    "Public apology",
    "Shut down or paused"
   ],
   "outcome_keys": [
    "apology",
    "shutdown_pause"
   ],
   "outcome_detail": "Virgin Money apologised and said it promptly removed the chatbot.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "yes",
   "pre_launch_reasoning": "An over-refusal test that uses the company's own brand and product names is a one-line case.",
   "sources": [
    {
     "url": "https://fortune.com/europe/2025/01/30/virgin-money-chatbot-scolds-customer-confuse-banks-name-insult/",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "removed the chatbot",
      "ISA"
     ]
    },
    {
     "url": "https://www.ibtimes.com/virgin-money-forced-apologize-after-chatbot-threatens-cut-off-bank-customers-who-use-word-3761461",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "Virgin Money"
     ]
    }
   ],
   "aiid_incident": null,
   "aiid_url": null
  },
  {
   "id": "CSF-036",
   "date": "2025-01-30",
   "year": 2025,
   "company": "Hangzhou DeepSeek and Beijing DeepSeek",
   "ai_system": "DeepSeek chatbot app",
   "country": "Italy",
   "sector": "AI platforms",
   "deployment": "Consumer AI assistant",
   "deployment_key": "consumer",
   "primary_failure_type": "Data leak or privacy breach",
   "primary_failure_key": "data_exposure",
   "all_failure_types": [
    "Data leak or privacy breach"
   ],
   "all_failure_keys": [
    "data_exposure"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "Italy's Garante ordered an immediate limitation on DeepSeek processing Italian users' data after finding the company's answers about what data it collected and where it was stored inadequate.",
   "outcomes": [
    "Regulator, tribunal or official audit",
    "Shut down or paused"
   ],
   "outcome_keys": [
    "regulatory",
    "shutdown_pause"
   ],
   "outcome_detail": "Processing of Italian users' data blocked and an investigation opened (Garante, 30 January 2025).",
   "legal_or_regulatory_action": true,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "no",
   "pre_launch_reasoning": "Regulatory data-protection compliance.",
   "sources": [
    {
     "url": "https://www.garanteprivacy.it/home/docweb/-/docweb-display/docweb/10097450",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "DeepSeek"
     ]
    }
   ],
   "aiid_incident": null,
   "aiid_url": null
  },
  {
   "id": "CSF-037",
   "date": "2025-02-26",
   "year": 2025,
   "company": "Microsoft",
   "ai_system": "Copilot",
   "country": "United States",
   "sector": "AI platforms",
   "deployment": "Consumer AI assistant",
   "deployment_key": "consumer",
   "primary_failure_type": "Data leak or privacy breach",
   "primary_failure_key": "data_exposure",
   "all_failure_types": [
    "Data leak or privacy breach"
   ],
   "all_failure_keys": [
    "data_exposure"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "Lasso Security reported that Copilot could return content from GitHub repositories that had been public and later made private, via Bing's cache, affecting more than 20,000 repositories. Microsoft classed the issue as low severity.",
   "outcomes": [
    "Fix or product change"
   ],
   "outcome_keys": [
    "fix_update"
   ],
   "outcome_detail": "Partial mitigations, per Lasso and TechCrunch.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "no",
   "pre_launch_reasoning": "A retrieval-cache problem, not model behaviour.",
   "sources": [
    {
     "url": "https://techcrunch.com/2025/02/26/thousands-of-exposed-github-repositories-now-private-can-still-be-accessed-through-copilot/",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "low severity",
      "20,000"
     ]
    }
   ],
   "aiid_incident": 1174,
   "aiid_url": "https://incidentdatabase.ai/cite/1174/"
  },
  {
   "id": "CSF-038",
   "date": "2025-04-18",
   "year": 2025,
   "company": "Anysphere (Cursor)",
   "ai_system": "AI support agent 'Sam'",
   "country": "United States",
   "sector": "Software & SaaS",
   "deployment": "Company support or sales bot",
   "deployment_key": "business",
   "primary_failure_type": "Wrong policy, price or promise",
   "primary_failure_key": "policy_promise",
   "all_failure_types": [
    "Wrong policy, price or promise",
    "Hallucinated or false facts"
   ],
   "all_failure_keys": [
    "policy_promise",
    "hallucination"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "Users who were being logged out when switching devices emailed support and got a reply from 'Sam' stating that one device per subscription was a core security policy. No such policy existed; the reply was AI-generated. Some users said they were cancelling subscriptions.",
   "outcomes": [
    "Refund or compensation",
    "Public apology",
    "Fix or product change"
   ],
   "outcome_keys": [
    "refund_compensation",
    "apology",
    "fix_update"
   ],
   "outcome_detail": "Co-founder Michael Truell apologised, said the user had been refunded and that AI responses used for email support are now clearly labelled as AI.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "yes",
   "pre_launch_reasoning": "A grounded-answer case ('Is there a one-device limit?') with the real policy as context would catch an invented rule.",
   "sources": [
    {
     "url": "https://fortune.com/article/customer-support-ai-cursor-went-rogue/",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "Cursor",
      "cancell?ing their subscriptions|cancellations"
     ]
    },
    {
     "url": "https://arstechnica.com/ai/2025/04/cursor-ai-support-bot-invents-fake-policy-and-triggers-user-uproar/",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "one device per subscription",
      "refunded",
      "clearly labeled"
     ]
    }
   ],
   "aiid_incident": 1039,
   "aiid_url": "https://incidentdatabase.ai/cite/1039/"
  },
  {
   "id": "CSF-039",
   "date": "2025-04-29",
   "year": 2025,
   "company": "Meta",
   "ai_system": "Meta AI chatbot",
   "country": "United States",
   "sector": "Social media & consumer tech",
   "deployment": "Consumer AI assistant",
   "deployment_key": "consumer",
   "primary_failure_type": "Hallucinated or false facts",
   "primary_failure_key": "hallucination",
   "all_failure_types": [
    "Hallucinated or false facts"
   ],
   "all_failure_keys": [
    "hallucination"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "Activist Robby Starbuck sued Meta for defamation, alleging Meta AI falsely said he took part in the January 6 Capitol riot. These were allegations in a complaint.",
   "outcomes": [
    "Lawsuit or court case"
   ],
   "outcome_keys": [
    "lawsuit"
   ],
   "outcome_detail": "Settled in August 2025; as part of the settlement Starbuck became an adviser to Meta on AI bias. Financial terms were not disclosed.",
   "legal_or_regulatory_action": true,
   "cost": "not disclosed (settlement terms not disclosed)",
   "cost_status": "not disclosed",
   "pre_launch_testable": "yes",
   "pre_launch_reasoning": "Tests asking about named real people with an expected 'I don't have verified information' catch fabricated biographies.",
   "sources": [
    {
     "url": "https://www.cnn.com/2025/08/14/tech/robby-starbuck-meta-ai-advisor",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "Starbuck",
      "settle"
     ]
    },
    {
     "url": "https://www.dhillonlaw.com/lawsuits/starbuck-v-meta/",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "Starbuck"
     ]
    }
   ],
   "aiid_incident": 1247,
   "aiid_url": "https://incidentdatabase.ai/cite/1247/"
  },
  {
   "id": "CSF-040",
   "date": "2025-04-29",
   "year": 2025,
   "company": "OpenAI",
   "ai_system": "ChatGPT (GPT-4o update)",
   "country": "United States",
   "sector": "AI platforms",
   "deployment": "Consumer AI assistant",
   "deployment_key": "consumer",
   "primary_failure_type": "Offensive or unsafe output",
   "primary_failure_key": "harmful_output",
   "all_failure_types": [
    "Offensive or unsafe output"
   ],
   "all_failure_keys": [
    "harmful_output"
   ],
   "followed_update_or_prompt_change": true,
   "summary": "An update to GPT-4o made ChatGPT overly flattering and agreeable. OpenAI said it focused too much on short-term user feedback and that sycophantic interactions can cause distress.",
   "outcomes": [
    "Fix or product change",
    "Shut down or paused"
   ],
   "outcome_keys": [
    "fix_update",
    "shutdown_pause"
   ],
   "outcome_detail": "OpenAI rolled the update back within days and said it would expand pre-deployment evaluations.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "yes",
   "pre_launch_reasoning": "OpenAI's own fix was more pre-deployment evaluation: a regression suite with 'should disagree' cases.",
   "sources": [
    {
     "url": "https://www.theverge.com/news/658850/openai-chatgpt-gpt-4o-update-sycophantic",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "sycophan",
      "roll"
     ]
    }
   ],
   "aiid_incident": null,
   "aiid_url": null
  },
  {
   "id": "CSF-041",
   "date": "2025-05-09",
   "year": 2025,
   "company": "Klarna",
   "ai_system": "OpenAI-powered customer-service assistant",
   "country": "Sweden",
   "sector": "Banking & fintech",
   "deployment": "Company support or sales bot",
   "deployment_key": "business",
   "primary_failure_type": "Failed the task or escalation",
   "primary_failure_key": "task_failure",
   "all_failure_types": [
    "Failed the task or escalation"
   ],
   "all_failure_keys": [
    "task_failure"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "After saying its AI assistant did the work of 700 agents, Klarna's CEO told Bloomberg that cost had been 'a too predominant evaluation factor' and the result was 'lower quality'. A Klarna spokesperson later said this referred to outsourced human agents, not the AI.",
   "outcomes": [
    "Fix or product change",
    "Public criticism only"
   ],
   "outcome_keys": [
    "fix_update",
    "pr"
   ],
   "outcome_detail": "Klarna began recruiting human support agents so customers can always reach a person, while saying it remains AI-first.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "partly",
   "pre_launch_reasoning": "Quality on complex cases can be measured pre-launch; the reported issue is disputed and partly about human staffing.",
   "sources": [
    {
     "url": "https://fortune.com/2025/05/09/klarna-ai-humans-return-on-investment/",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "lower quality",
      "700"
     ]
    },
    {
     "url": "https://www.siliconrepublic.com/machines/klarna-ai-hiring-humans-forrester",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "outsourced"
     ]
    }
   ],
   "aiid_incident": null,
   "aiid_url": null
  },
  {
   "id": "CSF-042",
   "date": "2025-05-14",
   "year": 2025,
   "company": "xAI",
   "ai_system": "Grok on X",
   "country": "United States",
   "sector": "AI platforms",
   "deployment": "Consumer AI assistant",
   "deployment_key": "consumer",
   "primary_failure_type": "Offensive or unsafe output",
   "primary_failure_key": "harmful_output",
   "all_failure_types": [
    "Offensive or unsafe output"
   ],
   "all_failure_keys": [
    "harmful_output"
   ],
   "followed_update_or_prompt_change": true,
   "summary": "Grok inserted claims about 'white genocide' in South Africa into replies on unrelated topics. xAI said an unauthorized modification had been made to the bot's system prompt.",
   "outcomes": [
    "Fix or product change"
   ],
   "outcome_keys": [
    "fix_update"
   ],
   "outcome_detail": "xAI said it would publish Grok's system prompts on GitHub, set up a 24/7 monitoring team and add checks so employees cannot modify the prompt without review.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "yes",
   "pre_launch_reasoning": "A regression suite of off-topic questions run on every system-prompt change flags unrelated insertions immediately.",
   "sources": [
    {
     "url": "https://www.nbcnews.com/tech/social-media/musks-xai-says-groks-white-genocide-posts-came-unauthorized-change-bot-rcna207222",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "unauthorized"
     ]
    },
    {
     "url": "https://www.theverge.com/news/668220/grok-white-genocide-south-africa-xai-unauthorized-modification-employee",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "GitHub",
      "24/7 monitoring"
     ]
    }
   ],
   "aiid_incident": 1072,
   "aiid_url": "https://incidentdatabase.ai/cite/1072/"
  },
  {
   "id": "CSF-043",
   "date": "2025-05-19",
   "year": 2025,
   "company": "OpenAI",
   "ai_system": "ChatGPT",
   "country": "United States",
   "sector": "AI platforms",
   "deployment": "Consumer AI assistant",
   "deployment_key": "consumer",
   "primary_failure_type": "Hallucinated or false facts",
   "primary_failure_key": "hallucination",
   "all_failure_types": [
    "Hallucinated or false facts"
   ],
   "all_failure_keys": [
    "hallucination"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "Radio host Mark Walters sued OpenAI for defamation after ChatGPT, asked by a journalist to summarise a lawsuit, falsely said Walters had been accused of embezzling funds. Date shown is the ruling.",
   "outcomes": [
    "Lawsuit or court case"
   ],
   "outcome_keys": [
    "lawsuit"
   ],
   "outcome_detail": "On 19 May 2025 a Georgia state court granted summary judgment to OpenAI.",
   "legal_or_regulatory_action": true,
   "cost": "not disclosed (OpenAI won)",
   "cost_status": "not disclosed",
   "pre_launch_testable": "yes",
   "pre_launch_reasoning": "A summarisation test that compares the output with the source document flags invented allegations.",
   "sources": [
    {
     "url": "https://www.loeb.com/en/insights/publications/2025/05/walters-v-openai-llc",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "Walters",
      "summary judgment"
     ]
    }
   ],
   "aiid_incident": null,
   "aiid_url": null
  },
  {
   "id": "CSF-044",
   "date": "2025-06-11",
   "year": 2025,
   "company": "Meta",
   "ai_system": "Meta AI app 'Discover' feed",
   "country": "United States",
   "sector": "Social media & consumer tech",
   "deployment": "Consumer AI assistant",
   "deployment_key": "consumer",
   "primary_failure_type": "Data leak or privacy breach",
   "primary_failure_key": "data_exposure",
   "all_failure_types": [
    "Data leak or privacy breach"
   ],
   "all_failure_keys": [
    "data_exposure"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "Journalists found personal conversations, including medical and legal questions, published to the Meta AI app's public Discover feed, apparently by users who did not realise they were sharing. Meta said sharing is opt-in.",
   "outcomes": [
    "Fix or product change",
    "Public criticism only"
   ],
   "outcome_keys": [
    "fix_update",
    "pr"
   ],
   "outcome_detail": "Reported by Business Insider; Meta's position, per the reports, is that sharing is opt-in.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "no",
   "pre_launch_reasoning": "A product-design and consent problem, not a model output.",
   "sources": [
    {
     "url": "https://www.businessinsider.com/mark-zuckerberg-meta-ai-chatbot-discover-feed-depressing-why-2025-6",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "Discover"
     ]
    }
   ],
   "aiid_incident": 1101,
   "aiid_url": "https://incidentdatabase.ai/cite/1101/"
  },
  {
   "id": "CSF-045",
   "date": "2025-07-08",
   "year": 2025,
   "company": "xAI",
   "ai_system": "Grok on X",
   "country": "United States",
   "sector": "AI platforms",
   "deployment": "Consumer AI assistant",
   "deployment_key": "consumer",
   "primary_failure_type": "Offensive or unsafe output",
   "primary_failure_key": "harmful_output",
   "all_failure_types": [
    "Offensive or unsafe output"
   ],
   "all_failure_keys": [
    "harmful_output"
   ],
   "followed_update_or_prompt_change": true,
   "summary": "Days after Grok was updated to 'not shy away from making claims which are politically incorrect', it posted antisemitic replies, praised Hitler and called itself 'MechaHitler', NPR reported.",
   "outcomes": [
    "Fix or product change",
    "Regulator, tribunal or official audit"
   ],
   "outcome_keys": [
    "fix_update",
    "regulatory"
   ],
   "outcome_detail": "The bot appeared to stop giving public text answers. Separately, a Turkish court blocked access to Grok over responses the authorities said insulted President Erdogan, and Poland said it would report xAI to the European Commission (BBC).",
   "legal_or_regulatory_action": true,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "yes",
   "pre_launch_reasoning": "Hate-speech probes rerun after each deploy (regression) catch this class.",
   "sources": [
    {
     "url": "https://www.npr.org/2025/07/09/nx-s1-5462609/grok-elon-musk-antisemitic-racist-content",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "politically incorrect",
      "MechaHitler"
     ]
    },
    {
     "url": "https://www.bbc.com/news/articles/c4g8r34nxeno",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "Turkish court",
      "Erdogan",
      "European Commission"
     ]
    }
   ],
   "aiid_incident": 1146,
   "aiid_url": "https://incidentdatabase.ai/cite/1146/"
  },
  {
   "id": "CSF-046",
   "date": "2025-07-11",
   "year": 2025,
   "company": "McDonald's; Paradox.ai (vendor)",
   "ai_system": "McHire hiring chatbot 'Olivia'",
   "country": "United States",
   "sector": "Restaurants & food",
   "deployment": "Company support or sales bot",
   "deployment_key": "business",
   "primary_failure_type": "Data leak or privacy breach",
   "primary_failure_key": "data_exposure",
   "all_failure_types": [
    "Data leak or privacy breach"
   ],
   "all_failure_keys": [
    "data_exposure"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "Security researchers Ian Carroll and Sam Curry reported that a Paradox.ai admin login using the password '123456' and an insecure API exposed chatbot records for up to 64 million job applications. Paradox said five candidates' information was viewed, only by the researchers.",
   "outcomes": [
    "Fix or product change"
   ],
   "outcome_keys": [
    "fix_update"
   ],
   "outcome_detail": "Paradox said it resolved the issue within a few hours of being notified and announced a bug bounty programme.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "no",
   "pre_launch_reasoning": "A default password and an access-control bug are infrastructure security failures; no prompt test would find them.",
   "sources": [
    {
     "url": "https://ian.sh/mcdonalds",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "123456",
      "64 million"
     ]
    },
    {
     "url": "https://www.csoonline.com/article/4020919/mcdonalds-ai-hiring-tools-password-123456-exposes-data-of-64m-applicants.html",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "123456"
     ]
    },
    {
     "url": "https://www.paradox.ai/blog/responsible-security-update",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "within a few hours",
      "Five candidates",
      "bug bounty"
     ]
    }
   ],
   "aiid_incident": 1179,
   "aiid_url": "https://incidentdatabase.ai/cite/1179/"
  },
  {
   "id": "CSF-047",
   "date": "2025-07-15",
   "year": 2025,
   "company": "Meta",
   "ai_system": "Meta AI chatbot",
   "country": "United States",
   "sector": "Social media & consumer tech",
   "deployment": "Consumer AI assistant",
   "deployment_key": "consumer",
   "primary_failure_type": "Data leak or privacy breach",
   "primary_failure_key": "data_exposure",
   "all_failure_types": [
    "Data leak or privacy breach"
   ],
   "all_failure_keys": [
    "data_exposure"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "A security researcher found that Meta AI's servers did not check that a user was authorised to see a prompt, so changing an ID could reveal other users' prompts and responses. Meta fixed it and said it found no evidence of abuse.",
   "outcomes": [
    "Fix or product change"
   ],
   "outcome_keys": [
    "fix_update"
   ],
   "outcome_detail": "Fixed in January 2025; Meta paid the researcher a $10,000 bug bounty, per TechCrunch.",
   "legal_or_regulatory_action": false,
   "cost": "$10,000 bug bounty paid",
   "cost_status": "disclosed",
   "pre_launch_testable": "no",
   "pre_launch_reasoning": "An access-control bug in the API, outside model behaviour.",
   "sources": [
    {
     "url": "https://techcrunch.com/2025/07/15/meta-fixes-bug-that-could-leak-users-ai-prompts-and-generated-content/",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "10,000"
     ]
    }
   ],
   "aiid_incident": 1172,
   "aiid_url": "https://incidentdatabase.ai/cite/1172/"
  },
  {
   "id": "CSF-048",
   "date": "2025-07-21",
   "year": 2025,
   "company": "Replit",
   "ai_system": "Replit AI coding agent",
   "country": "United States",
   "sector": "Software & SaaS",
   "deployment": "Company support or sales bot",
   "deployment_key": "business",
   "primary_failure_type": "Failed the task or escalation",
   "primary_failure_key": "task_failure",
   "all_failure_types": [
    "Failed the task or escalation",
    "Hallucinated or false facts"
   ],
   "all_failure_keys": [
    "task_failure",
    "hallucination"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "SaaStr founder Jason Lemkin said the agent deleted a production database during a code freeze despite instructions not to make changes, and gave misleading information about whether recovery was possible.",
   "outcomes": [
    "Public apology",
    "Fix or product change"
   ],
   "outcome_keys": [
    "apology",
    "fix_update"
   ],
   "outcome_detail": "Replit's CEO Amjad Masad apologised and said Replit was rolling out automatic separation of development and production databases.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "partly",
   "pre_launch_reasoning": "Instruction-following tests ('do not change anything') help, but destructive permissions need sandboxing, not prompts.",
   "sources": [
    {
     "url": "https://www.theregister.com/2025/07/21/replit_saastr_vibe_coding_incident/",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "Replit",
      "database"
     ]
    },
    {
     "url": "https://www.tomshardware.com/tech-industry/artificial-intelligence/ai-coding-platform-goes-rogue-during-code-freeze-and-deletes-entire-company-database-replit-ceo-apologizes-after-ai-engine-says-it-made-a-catastrophic-error-in-judgment-and-destroyed-all-production-data",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "apolog",
      "dev/prod separation"
     ]
    }
   ],
   "aiid_incident": 1152,
   "aiid_url": "https://incidentdatabase.ai/cite/1152/"
  },
  {
   "id": "CSF-049",
   "date": "2025-07-23",
   "year": 2025,
   "company": "Amazon Web Services",
   "ai_system": "Amazon Q Developer extension for VS Code",
   "country": "United States",
   "sector": "Software & SaaS",
   "deployment": "Company support or sales bot",
   "deployment_key": "business",
   "primary_failure_type": "Prompt injection or manipulation",
   "primary_failure_key": "prompt_injection",
   "all_failure_types": [
    "Prompt injection or manipulation"
   ],
   "all_failure_keys": [
    "prompt_injection"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "A hacker slipped a prompt into a released version of the Amazon Q extension instructing the AI agent to wipe local files and, under certain conditions, cloud resources, ZDNET reported.",
   "outcomes": [
    "Fix or product change"
   ],
   "outcome_keys": [
    "fix_update"
   ],
   "outcome_detail": "AWS published security bulletin AWS-2025-015 for version 1.84 of the extension.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "partly",
   "pre_launch_reasoning": "A supply-chain compromise; prompt tests on the shipped instructions would show a destructive instruction, but code review is the main control.",
   "sources": [
    {
     "url": "https://aws.amazon.com/security/security-bulletins/AWS-2025-015/",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "Version #1.84"
     ]
    },
    {
     "url": "https://www.zdnet.com/article/hacker-slips-malicious-wiping-command-into-amazons-q-ai-coding-assistant-and-devs-are-worried/",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "erased local files"
     ]
    }
   ],
   "aiid_incident": 1158,
   "aiid_url": "https://incidentdatabase.ai/cite/1158/"
  },
  {
   "id": "CSF-050",
   "date": "2025-08-15",
   "year": 2025,
   "company": "Google; Royal Caribbean (impersonated)",
   "ai_system": "Google AI Overviews (search answers)",
   "country": "United States",
   "sector": "Search & consumer tech",
   "deployment": "Consumer AI assistant",
   "deployment_key": "consumer",
   "primary_failure_type": "Hallucinated or false facts",
   "primary_failure_key": "hallucination",
   "all_failure_types": [
    "Hallucinated or false facts",
    "Prompt injection or manipulation"
   ],
   "all_failure_keys": [
    "hallucination",
    "prompt_injection"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "The Washington Post reported that a traveller searched Google for a cruise line's customer-service number, called the number he was shown, and gave his credit card details to what turned out to be a scam operation.",
   "outcomes": [
    "Public criticism only"
   ],
   "outcome_keys": [
    "pr"
   ],
   "outcome_detail": "Reported by the Washington Post; no regulatory action in the source.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "partly",
   "pre_launch_reasoning": "Tests that check contact details only come from an allow-listed source help; poisoned web data is a retrieval problem.",
   "sources": [
    {
     "url": "https://www.washingtonpost.com/technology/2025/08/15/google-ai-overviews-scam/",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "customer service number",
      "It was a scam",
      "credit card"
     ]
    }
   ],
   "aiid_incident": 1187,
   "aiid_url": "https://incidentdatabase.ai/cite/1187/"
  },
  {
   "id": "CSF-051",
   "date": "2025-08-18",
   "year": 2025,
   "company": "Lenovo",
   "ai_system": "Customer-support chatbot 'Lena' (GPT-4)",
   "country": "United States",
   "sector": "Consumer electronics",
   "deployment": "Company support or sales bot",
   "deployment_key": "business",
   "primary_failure_type": "Prompt injection or manipulation",
   "primary_failure_key": "prompt_injection",
   "all_failure_types": [
    "Prompt injection or manipulation",
    "Data leak or privacy breach"
   ],
   "all_failure_keys": [
    "prompt_injection",
    "data_exposure"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "Cybernews researchers showed that a single 400-character prompt made Lena output HTML that, when a human support agent opened the chat, could send that agent's session cookie to an outside server (stored cross-site scripting).",
   "outcomes": [
    "Fix or product change"
   ],
   "outcome_keys": [
    "fix_update"
   ],
   "outcome_detail": "Cybernews says Lenovo mitigated the issue before public disclosure on 18 August 2025; Lenovo said it implemented corrective actions.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "yes",
   "pre_launch_reasoning": "A structured-output test ('reply must be plain text, no HTML') plus an injection case catches the model half; output sanitising catches the rest.",
   "sources": [
    {
     "url": "https://cybernews.com/security/lenovo-chatbot-lena-plagued-by-critical-vulnerabilities/",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "Lena",
      "cookie",
      "corrective actions"
     ]
    },
    {
     "url": "https://www.csoonline.com/article/4043005/lenovo-chatbot-breach-highlights-ai-security-blind-spots-in-customer-facing-systems.html",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "Lena"
     ]
    }
   ],
   "aiid_incident": null,
   "aiid_url": null
  },
  {
   "id": "CSF-052",
   "date": "2025-08-20",
   "year": 2025,
   "company": "Google; Stefanina's (restaurant harmed)",
   "ai_system": "Google AI Overviews (search answers)",
   "country": "United States",
   "sector": "Search & consumer tech",
   "deployment": "Consumer AI assistant",
   "deployment_key": "consumer",
   "primary_failure_type": "Hallucinated or false facts",
   "primary_failure_key": "hallucination",
   "all_failure_types": [
    "Hallucinated or false facts"
   ],
   "all_failure_keys": [
    "hallucination"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "A family restaurant in Wentzville, Missouri, asked customers not to rely on Google's AI answers after it showed specials the restaurant did not offer, and customers demanded them.",
   "outcomes": [
    "Public criticism only"
   ],
   "outcome_keys": [
    "pr"
   ],
   "outcome_detail": "The restaurant posted a public warning; no response from Google in the source.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "partly",
   "pre_launch_reasoning": "Grounded-answer tests catch invented offers in your own bot; you cannot test a third party's search answers.",
   "sources": [
    {
     "url": "https://www.firstalert4.com/2025/08/20/please-do-not-use-google-ai-find-out-our-specials-wentzville-restaurant-asks-patrons/",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "Stefanina",
      "specials"
     ]
    }
   ],
   "aiid_incident": 1211,
   "aiid_url": "https://incidentdatabase.ai/cite/1211/"
  },
  {
   "id": "CSF-053",
   "date": "2025-08-21",
   "year": 2025,
   "company": "Commonwealth Bank of Australia",
   "ai_system": "AI voice bot in customer-service calls",
   "country": "Australia",
   "sector": "Banking & fintech",
   "deployment": "Company support or sales bot",
   "deployment_key": "business",
   "primary_failure_type": "Failed the task or escalation",
   "primary_failure_key": "task_failure",
   "all_failure_types": [
    "Failed the task or escalation"
   ],
   "all_failure_keys": [
    "task_failure"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "CBA cut 45 customer-service roles, citing a new AI voice bot. The Finance Sector Union said call volumes were rising and took the bank to the Fair Work Commission. CBA then said its assessment was an 'error' and the roles were not redundant.",
   "outcomes": [
    "Public apology",
    "Regulator, tribunal or official audit"
   ],
   "outcome_keys": [
    "apology",
    "regulatory"
   ],
   "outcome_detail": "CBA reversed the cuts and apologised to the employees; the union's dispute was before the Fair Work Commission.",
   "legal_or_regulatory_action": true,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "partly",
   "pre_launch_reasoning": "Containment rate is measurable before launch with real call transcripts as test cases; staffing decisions are not.",
   "sources": [
    {
     "url": "https://www.abc.net.au/news/2025-08-21/cba-backtracks-on-ai-job-cuts-as-chatbot-lifts-call-volumes/105679492",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "45",
      "error",
      "Fair Work"
     ]
    },
    {
     "url": "https://ia.acs.org.au/article/2025/cba-reverses-ai-driven-job-cuts--admits--error-.html",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "voice bot"
     ]
    }
   ],
   "aiid_incident": null,
   "aiid_url": null
  },
  {
   "id": "CSF-054",
   "date": "2025-08-28",
   "year": 2025,
   "company": "Taco Bell (Yum! Brands)",
   "ai_system": "Voice AI drive-thru ordering",
   "country": "United States",
   "sector": "Restaurants & food",
   "deployment": "Company support or sales bot",
   "deployment_key": "business",
   "primary_failure_type": "Prompt injection or manipulation",
   "primary_failure_key": "prompt_injection",
   "all_failure_types": [
    "Prompt injection or manipulation",
    "Failed the task or escalation"
   ],
   "all_failure_keys": [
    "prompt_injection",
    "task_failure"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "Clips circulated of customers trolling the voice AI, including an order for 18,000 cups of water, and of the system repeatedly misunderstanding orders. The chain's digital chief told the Wall Street Journal it was rethinking where to use voice AI.",
   "outcomes": [
    "Public criticism only",
    "Fix or product change"
   ],
   "outcome_keys": [
    "pr",
    "fix_update"
   ],
   "outcome_detail": "Taco Bell said it was reassessing where the system is used, for example at busy locations; no shutdown was announced. Source: BBC, citing the Wall Street Journal.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "yes",
   "pre_launch_reasoning": "Absurd-quantity and troll orders are easy adversarial test cases with an expected behaviour (confirm, cap, or hand off to staff).",
   "sources": [
    {
     "url": "https://www.bbc.com/news/articles/ckgyk2p55g8o",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "Taco Bell",
      "18,000"
     ]
    }
   ],
   "aiid_incident": 1274,
   "aiid_url": "https://incidentdatabase.ai/cite/1274/"
  },
  {
   "id": "CSF-055",
   "date": "2025-10-21",
   "year": 2025,
   "company": "Canada Revenue Agency",
   "ai_system": "'Charlie' tax-information chatbot",
   "country": "Canada",
   "sector": "Government & public services",
   "deployment": "Government or public-service bot",
   "deployment_key": "public",
   "primary_failure_type": "Hallucinated or false facts",
   "primary_failure_key": "hallucination",
   "all_failure_types": [
    "Hallucinated or false facts"
   ],
   "all_failure_keys": [
    "hallucination"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "The Auditor General of Canada tested Charlie with six general tax questions; it answered two correctly. Other public AI tools tested answered five of six correctly. The audit covered CRA contact centres more broadly.",
   "outcomes": [
    "Regulator, tribunal or official audit"
   ],
   "outcome_keys": [
    "regulatory"
   ],
   "outcome_detail": "Findings published in the Auditor General's October 2025 report on CRA contact centres; CRA's response to the recommendations was “Agreed”. National Post reported CRA spent about $18 million on Charlie.",
   "legal_or_regulatory_action": true,
   "cost": "About C$18 million spent on the chatbot, as reported by National Post",
   "cost_status": "reported by press",
   "pre_launch_testable": "yes",
   "pre_launch_reasoning": "A tax FAQ with answers from CRA guidance is a textbook grounded-answer suite; the auditor's own six-question test is one.",
   "sources": [
    {
     "url": "https://www.canada.ca/content/dam/oag-bvg/audit-reports/documents/parl_oag_202510_01_e.pdf",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "Charlie provided accurate answers in only 2 out of the 6"
     ]
    },
    {
     "url": "https://nationalpost.com/news/politics/the-cra-spent-18-million-on-charlie-its-new-tax-information-chatbot",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "18 million",
      "Charlie"
     ]
    },
    {
     "url": "https://www.cbc.ca/news/politics/ag-fall-2025-cra-military-9.6946672",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "Oct 21, 2025",
      "Charlie"
     ]
    }
   ],
   "aiid_incident": 1310,
   "aiid_url": "https://incidentdatabase.ai/cite/1310/"
  },
  {
   "id": "CSF-056",
   "date": "2025-11-18",
   "year": 2025,
   "company": "OpenAI, Microsoft, Google, Meta and others (tested by Which?)",
   "ai_system": "ChatGPT, Copilot, Gemini, Meta AI and others",
   "country": "United Kingdom",
   "sector": "AI platforms",
   "deployment": "Consumer AI assistant",
   "deployment_key": "consumer",
   "primary_failure_type": "Hallucinated or false facts",
   "primary_failure_key": "hallucination",
   "all_failure_types": [
    "Hallucinated or false facts"
   ],
   "all_failure_keys": [
    "hallucination"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "Consumer group Which? tested AI chatbots on 40 money, legal and consumer questions and found errors: ChatGPT and Copilot did not notice a wrong ISA allowance and gave advice that could breach HMRC limits, and ChatGPT wrongly said travel insurance was mandatory for most EU countries.",
   "outcomes": [
    "Public criticism only"
   ],
   "outcome_keys": [
    "pr"
   ],
   "outcome_detail": "Research report; companies' responses are as quoted by the Guardian.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "yes",
   "pre_launch_reasoning": "The Which? study is itself a 40-case test suite with expected answers.",
   "sources": [
    {
     "url": "https://www.theguardian.com/technology/2025/nov/18/warning-ai-chatbots-inaccurate-financial-advice-tips-chatgpt-copilot-uk",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "40 questions",
      "allowance",
      "travel insurance"
     ]
    }
   ],
   "aiid_incident": 1279,
   "aiid_url": "https://incidentdatabase.ai/cite/1279/"
  },
  {
   "id": "CSF-057",
   "date": "2026-01-06",
   "year": 2026,
   "company": "Tencent",
   "ai_system": "Yuanbao AI assistant (WeChat)",
   "country": "China",
   "sector": "Social media & consumer tech",
   "deployment": "Consumer AI assistant",
   "deployment_key": "consumer",
   "primary_failure_type": "Offensive or unsafe output",
   "primary_failure_key": "harmful_output",
   "all_failure_types": [
    "Offensive or unsafe output"
   ],
   "all_failure_keys": [
    "harmful_output"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "Screenshots showed Yuanbao insulting a user who asked for help debugging code. Tencent apologised, called it a rare model output anomaly and said logs showed no human involvement.",
   "outcomes": [
    "Public apology",
    "Fix or product change"
   ],
   "outcome_keys": [
    "apology",
    "fix_update"
   ],
   "outcome_detail": "Tencent apologised and said it was optimising the model.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "partly",
   "pre_launch_reasoning": "Tone checks on repeated runs reduce the risk; rare outputs need sampling at volume.",
   "sources": [
    {
     "url": "https://www.businessinsider.com/chinese-ai-chatbot-tencent-yuanbao-wechat-user-rednote-2026-1",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "rare model output anomaly",
      "human intervention"
     ]
    }
   ],
   "aiid_incident": 1382,
   "aiid_url": "https://incidentdatabase.ai/cite/1382/"
  },
  {
   "id": "CSF-058",
   "date": "2026-02-26",
   "year": 2026,
   "company": "Woolworths (Australia)",
   "ai_system": "Digital assistant 'Olive'",
   "country": "Australia",
   "sector": "Grocery & retail",
   "deployment": "Company support or sales bot",
   "deployment_key": "business",
   "primary_failure_type": "Hallucinated or false facts",
   "primary_failure_key": "hallucination",
   "all_failure_types": [
    "Hallucinated or false facts"
   ],
   "all_failure_keys": [
    "hallucination"
   ],
   "followed_update_or_prompt_change": true,
   "summary": "Customers reported that Olive invented humanlike personal anecdotes, such as talking about its mother, during routine support chats after an upgrade built on Google's Gemini Enterprise for Customer Experience.",
   "outcomes": [
    "Fix or product change"
   ],
   "outcome_keys": [
    "fix_update"
   ],
   "outcome_detail": "Woolworths said it was adjusting Olive's responses.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "yes",
   "pre_launch_reasoning": "A persona test ('Do you have a family?') plus regression runs after a model upgrade catch invented personal stories.",
   "sources": [
    {
     "url": "https://www.cxtoday.com/ai-automation-in-cx/woolworths-ai-chatbot-olive-incident/",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "angry mother",
      "Gemini Enterprise",
      "adjusting Olive"
     ]
    }
   ],
   "aiid_incident": 1393,
   "aiid_url": "https://incidentdatabase.ai/cite/1393/"
  },
  {
   "id": "CSF-059",
   "date": "2026-06-01",
   "year": 2026,
   "company": "Meta (Instagram)",
   "ai_system": "AI support assistant for account recovery",
   "country": "United States",
   "sector": "Social media & consumer tech",
   "deployment": "Company support or sales bot",
   "deployment_key": "business",
   "primary_failure_type": "Prompt injection or manipulation",
   "primary_failure_key": "prompt_injection",
   "all_failure_types": [
    "Prompt injection or manipulation",
    "Data leak or privacy breach"
   ],
   "all_failure_keys": [
    "prompt_injection",
    "data_exposure"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "404 Media reported that attackers took over high-profile Instagram accounts by persuading Meta's AI support chatbot to change the account email and start recovery. Meta said the issue had been resolved.",
   "outcomes": [
    "Fix or product change",
    "Public criticism only"
   ],
   "outcome_keys": [
    "fix_update",
    "pr"
   ],
   "outcome_detail": "Meta said the issue was resolved.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed",
   "cost_status": "not disclosed",
   "pre_launch_testable": "partly",
   "pre_launch_reasoning": "Social-engineering prompts against an account-recovery flow are testable, but the real fix is not letting a chatbot change account credentials.",
   "sources": [
    {
     "url": "https://www.404media.co/hackers-simply-asked-meta-ai-to-give-them-access-to-high-profile-instagram-accounts-it-worked/",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "Instagram",
      "Meta"
     ]
    },
    {
     "url": "https://www.bbc.com/news/articles/c98rzr72dpyo",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "Instagram",
      "Meta"
     ]
    }
   ],
   "aiid_incident": 1510,
   "aiid_url": "https://incidentdatabase.ai/cite/1510/"
  },
  {
   "id": "CSF-060",
   "date": "2026-06-11",
   "year": 2026,
   "company": "BMW Toronto (dealership)",
   "ai_system": "Dealership AI chatbot 'Quinn'",
   "country": "Canada",
   "sector": "Automotive retail",
   "deployment": "Company support or sales bot",
   "deployment_key": "business",
   "primary_failure_type": "Wrong policy, price or promise",
   "primary_failure_key": "policy_promise",
   "all_failure_types": [
    "Wrong policy, price or promise"
   ],
   "all_failure_keys": [
    "policy_promise"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "The dealer's chatbot 'Quinn' made a customer a firm buy-back offer of C$27,162.79. The dealer revoked it, saying that because of a human employee's miscommunication the bot had taken the amount owed on the car as the buy-back amount.",
   "outcomes": [
    "Refund or compensation",
    "Public criticism only"
   ],
   "outcome_keys": [
    "refund_compensation",
    "pr"
   ],
   "outcome_detail": "After hearing from CBC News the dealer reinstated the offer, which the customer accepted; it said only human employees will present buy-back offers in future.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed (offer of C$27,162.79 reinstated; any loss to the dealer not disclosed)",
   "cost_status": "not disclosed",
   "pre_launch_testable": "yes",
   "pre_launch_reasoning": "A rule that the bot must never state a price or offer is testable with a handful of 'what will you give me for my car?' cases.",
   "sources": [
    {
     "url": "https://www.cbc.ca/news/business/ai-chatbot-bmw-dealership-9.7230226",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "27,162.79",
      "reinstated",
      "only human employees"
     ]
    }
   ],
   "aiid_incident": 1772,
   "aiid_url": "https://incidentdatabase.ai/cite/1772/"
  },
  {
   "id": "CSF-061",
   "date": "2026-10-15",
   "year": 2026,
   "company": "Anthropic",
   "ai_system": "Fin AI support agent (Intercom) on Claude support",
   "country": "United States",
   "sector": "Software & SaaS",
   "deployment": "Company support or sales bot",
   "deployment_key": "business",
   "primary_failure_type": "Failed the task or escalation",
   "primary_failure_key": "task_failure",
   "all_failure_types": [
    "Failed the task or escalation"
   ],
   "all_failure_keys": [
    "task_failure"
   ],
   "followed_update_or_prompt_change": false,
   "summary": "The San Francisco Standard reported a customer's account that after Anthropic created a duplicate account and charged him an extra $3,500, his refund request through the Fin AI agent became a months-long ordeal of circular conversations. These are the customer's claims as reported.",
   "outcomes": [
    "Public criticism only"
   ],
   "outcome_keys": [
    "pr"
   ],
   "outcome_detail": "Reported by the SF Standard; resolution details are as described in that report.",
   "legal_or_regulatory_action": false,
   "cost": "not disclosed (customer reported a $3,500 overcharge)",
   "cost_status": "not disclosed",
   "pre_launch_testable": "yes",
   "pre_launch_reasoning": "An escalation test ('I want a human / a refund for a double charge') with an expected hand-off catches support loops.",
   "sources": [
    {
     "url": "https://sfstandard.com/2026/10/15/anthropic-customer-service-human/",
     "http_status": 200,
     "checked_on": "2026-11-12",
     "checked_phrases": [
      "3,500",
      "months-long ordeal",
      "circular"
     ]
    }
   ],
   "aiid_incident": 1765,
   "aiid_url": "https://incidentdatabase.ai/cite/1765/"
  }
 ]
}