{
  "page": {
    "id": "chfw0vm4psbx",
    "name": "StreamShark",
    "url": "https://status.streamshark.io"
  },
  "status": {
    "indicator": "none",
    "description": "All Systems Operational"
  },
  "components": [{"status":"operational","name":"Web Portal","created_at":"2024-11-11T04:17:29.199Z","updated_at":"2025-06-12T21:10:41.692Z","position":1,"description":null,"showcase":false,"start_date":"2024-11-11T00:00:00.000Z","id":"v15spklppw57","group_id":null,"page_id":"chfw0vm4psbx"},{"status":"operational","name":"Streaming Events","created_at":"2024-11-13T04:03:15.137Z","updated_at":"2025-06-12T20:45:58.186Z","position":2,"description":null,"showcase":false,"start_date":"2024-11-13T00:00:00.000Z","id":"srd5cfcrm9qb","group_id":null,"page_id":"chfw0vm4psbx"},{"status":"operational","name":"Video on Demand","created_at":"2022-02-22T05:56:14.890Z","updated_at":"2026-03-22T08:10:07.183Z","position":3,"description":null,"showcase":false,"start_date":"2022-02-22T00:00:00.000Z","id":"sgbqpgt5bvl2","group_id":null,"page_id":"chfw0vm4psbx"},{"status":"operational","name":"Live Scheduler","created_at":"2024-11-13T04:12:48.172Z","updated_at":"2026-03-22T08:10:07.197Z","position":4,"description":null,"showcase":false,"start_date":"2024-11-13T00:00:00.000Z","id":"wb2bk2wx03y3","group_id":null,"page_id":"chfw0vm4psbx"},{"status":"operational","name":"Video Player","created_at":"2022-02-22T05:54:38.442Z","updated_at":"2026-05-21T07:01:49.221Z","position":5,"description":null,"showcase":false,"start_date":"2022-02-22T00:00:00.000Z","id":"vrbgfn5cjnkg","group_id":null,"page_id":"chfw0vm4psbx"},{"status":"operational","name":"Storage","created_at":"2022-02-22T05:55:38.090Z","updated_at":"2026-03-22T08:10:07.211Z","position":6,"description":null,"showcase":false,"start_date":"2022-02-22T00:00:00.000Z","id":"vwbvg0ccxq2q","group_id":null,"page_id":"chfw0vm4psbx"}],
  "incidents": [{"name":"New video player unavailable with some existing video player templates","status":"resolved","created_at":"2026-05-21T00:54:22.000Z","updated_at":"2026-05-22T06:08:47.449Z","monitoring_at":"2026-05-21T03:29:28.775Z","resolved_at":"2026-05-21T07:01:49.000Z","impact":"major","shortlink":"https://stspg.io/w0rs5tyfr642","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":null,"scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2026-05-21T00:54:22.000Z","reminder_intervals":null,"id":"gf6zm7w1y6jr","page_id":"chfw0vm4psbx","incident_updates":[{"status":"resolved","body":"On 21 May at 6:54 am UTC, engineers completed all internal tests and verified the new player is back to normal functionality. We apologise for any disruption experienced by customers due to this issue. A post mortem report will be available in the next 24 hours.","created_at":"2026-05-21T07:01:49.249Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-05-22T06:03:58.641Z","display_at":"2026-05-21T07:01:49.000Z","deliver_notifications":true,"tweet_id":null,"id":"mbtmht9dgn5q","incident_id":"gf6zm7w1y6jr","custom_tweet":null,"affected_components":[{"code":"vrbgfn5cjnkg","name":"Video Player","old_status":"partial_outage","new_status":"operational"}]},{"status":"monitoring","body":"Engineers have deployed the fix and internal testing shows the issue as resolved. We will continue to monitor and close the incident if no further issues are detected.","created_at":"2026-05-21T03:29:28.775Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-05-21T03:29:28.775Z","display_at":"2026-05-21T03:29:28.775Z","deliver_notifications":true,"tweet_id":null,"id":"tq3x8v199hfl","incident_id":"gf6zm7w1y6jr","custom_tweet":null,"affected_components":[{"code":"vrbgfn5cjnkg","name":"Video Player","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"identified","body":"Our engineers have identified the cause of the new video player failing to load. Certain existing video templates contain a missing property value which prevents the new player from loading correctly. Newly created templates and the default template are not affected by this issue\n\nWorkaround: If you are experiencing this issue, switching to the default template or creating a new template will resolve it. The legacy video player also remains fully operational.\n\nOur team is working on a fix to ensure all existing templates are compatible with the new player. We will post an update once a resolution has been deployed - ETA is ~1 hour","created_at":"2026-05-21T01:03:13.725Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-05-21T01:03:13.725Z","display_at":"2026-05-21T01:03:13.725Z","deliver_notifications":true,"tweet_id":null,"id":"xsqddgsc7kl2","incident_id":"gf6zm7w1y6jr","custom_tweet":null,"affected_components":[{"code":"vrbgfn5cjnkg","name":"Video Player","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"On 20 May at 11:43 pm UTC, our team identified an issue where the new video player is failing to load due to a compatibility issue with some existing video player templates. Our team has identified the cause and is actively working on a fix.\n\nIn the meantime, users are encouraged to use the legacy player if they are unable to load the new player.","created_at":"2026-05-21T00:54:22.903Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-05-22T06:03:22.717Z","display_at":"2026-05-21T00:54:22.000Z","deliver_notifications":true,"tweet_id":null,"id":"yzwk2c4q1ncd","incident_id":"gf6zm7w1y6jr","custom_tweet":null,"affected_components":[{"code":"vrbgfn5cjnkg","name":"Video Player","old_status":"operational","new_status":"partial_outage"}]}],"postmortem_body":"### Incident Impact\n\nOn 20 May 2026 UTC, a small number of customers experienced issues accessing certain New Player pages, which were unable to load correctly.\n\nWe have since analysed every impacted playback session and can confirm that only a very small number of customers were affected. The vast majority of customers experienced no disruption, and all other StreamShark services remained fully operational throughout.\n\n### Overview\n\nThe issue was introduced as part of a recent update that added new visual customisation options to the New Player. This update inadvertently caused certain older player configurations to become incompatible, resulting in those pages being unable to display correctly. The issue was not caught prior to release as our testing at the time did not cover the older configurations in use by our customers.\n\n### Detection\n\nThe issue was identified by a member of the StreamShark team during an internal testing. Upon detection, our engineering team was immediately engaged and began working to investigate and resolve the problem. The [StreamShark status page](https://status.streamshark.io/) was updated during the issue.\n\n* **First Detected:** Wednesday, 20 May 2026 at 11:43 p.m. UTC\n* **Mitigated:** Thursday, 21 May 2026 at 03:29 a.m. UTC\n* **Ongoing Monitoring:** Thursday, 21 May 2026 at 03:30 a.m. UTC\n\n### Root Cause\n\nThe issue stemmed from a compatibility gap between a recently introduced feature and certain older player configurations. When the update was released, a safeguard that should have accounted for these older configurations was not in place. As testing had been conducted using newer configurations only, this gap was not identified before the release went live.\n\n### Remediation\n\nThe root cause has been identified and fully resolved. All affected New Player pages have been restored and confirmed to be working correctly.\n\nWe understand how important reliability is to our customers, and we are taking the following steps to ensure this does not happen again:\n\n1. **Broader testing coverage** — We have updated our testing process to validate new releases against a wider range of customer configurations, including older ones, so that compatibility issues are caught before they reach production.\n2. **Automated safeguards** — We are introducing additional automated checks that will help detect potential issues earlier in the development process.","postmortem_body_last_updated_at":"2026-05-22T06:08:47.435Z","postmortem_ignored":false,"postmortem_published_at":"2026-05-22T06:08:47.435Z","postmortem_notified_subscribers":true,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Video Player","created_at":"2022-02-22T05:54:38.442Z","updated_at":"2026-05-21T07:01:49.221Z","position":5,"description":null,"showcase":false,"start_date":"2022-02-22T00:00:00.000Z","id":"vrbgfn5cjnkg","page_id":"chfw0vm4psbx","group_id":null}]},{"name":"Hardware Encoder polling issue","status":"resolved","created_at":"2026-03-25T21:59:28.000Z","updated_at":"2026-03-27T03:45:59.058Z","monitoring_at":null,"resolved_at":"2026-03-25T22:21:00.000Z","impact":"none","shortlink":"https://stspg.io/2g8vk4k0h81f","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":null,"scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2026-03-25T21:59:28.000Z","reminder_intervals":"[]","id":"hhbd0wzm2hw2","page_id":"chfw0vm4psbx","incident_updates":[{"status":"resolved","body":"The issue was identified as a problem with the cloud provider’s task queue service, which caused tasks to not be processed correctly. This has now been resolved, and operations have returned to normal. Preventative measures will be implemented to mitigate similar incidents in the future, and a detailed post-mortem will be shared soon.","created_at":"2026-03-26T07:05:00.016Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-03-26T07:10:32.571Z","display_at":"2026-03-25T22:21:00.000Z","deliver_notifications":false,"tweet_id":null,"id":"h4h4732h8pln","incident_id":"hhbd0wzm2hw2","custom_tweet":null,"affected_components":[{"code":"srd5cfcrm9qb","name":"Streaming Events","old_status":"operational","new_status":"operational"}]},{"status":"investigating","body":"We have observed potential polling issues with the Hardware Encoder that may be delaying operation processing.","created_at":"2026-03-26T07:04:28.875Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-03-26T07:07:33.005Z","display_at":"2026-03-25T21:59:28.000Z","deliver_notifications":false,"tweet_id":null,"id":"shw67p802mqq","incident_id":"hhbd0wzm2hw2","custom_tweet":null,"affected_components":[{"code":"srd5cfcrm9qb","name":"Streaming Events","old_status":"operational","new_status":"operational"}]}],"postmortem_body":"## **Incident Impact**\n\nHardware Encoder commands generated for managed hardware encoders could not be sent out to hardware encoders. This affected the following workflows:\n\n* Hardware encoder provisioning \\(for event create\\)\n* Hardware encoder deprovisioning \\(for event delete\\)\n* Hardware encoder stream start\n* Hardware encoder stream stop\n\nThe incident impact was from approximately 2:59 PM PDT to 3:21 PM PDT on the 25th of March 2026.\n\n## **Overview**\n\nStreamShark makes use of Google Cloud infrastructure to manage communications between our system and managed hardware encoders. When customers perform actions via the StreamShark portal — such as creating or deleting events, or starting and stopping streams — commands are generated and placed into Google managed queues \\(Cloud Tasks\\) for delivery to the corresponding encoder.\n\nA transient error in the Google Cloud Tasks service prevented our system from submitting commands into these queues. As a result:\n\n* Hardware encoder commands could not be delivered. This prevented tasks dependent on encoders starting \\(e.g. Location Session starts\\) from being actioned.\n* The Hardware Encoder management page incorrectly showed all encoders as down \\(not polling\\).\n\n## **Detection**\n\nThe issue was identified following customer reports that recording sessions linked to managed hardware encoders in the Locations product could not be started.\n\n## **Root Cause Detail**\n\nA sustained transient error in the Google Cloud Tasks service prevented our system from submitting commands into the queue. Our system automatically retries transient errors with backoff, but in this case the errors persisted for the duration of the incident.\n\nThe issue resolved itself at approximately 3:21 PM PDT and normal command processing resumed shortly after. Encoder status reporting in the portal was temporarily delayed during recovery but was restored to an accurate state.\n\n## **Remediation**\n\nWe are taking the following steps to prevent this type of incident from occurring again and to improve recovery speed:\n\n* **Improved monitoring and alerting** — we are implementing tighter alerting thresholds to detect queue processing issues earlier and reduce time to resolution.\n* **Infrastructure resilience** — we are adding safeguards to ensure our backend services remain available during transient upstream issues, reducing recovery time.\n* **Fallback processing paths** — we are investigating alternative approaches to command delivery and encoder status reporting so that a single upstream dependency failure does not block these workflows.","postmortem_body_last_updated_at":"2026-03-27T03:45:59.029Z","postmortem_ignored":false,"postmortem_published_at":"2026-03-27T03:45:59.029Z","postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Streaming Events","created_at":"2024-11-13T04:03:15.137Z","updated_at":"2025-06-12T20:45:58.186Z","position":2,"description":null,"showcase":false,"start_date":"2024-11-13T00:00:00.000Z","id":"srd5cfcrm9qb","page_id":"chfw0vm4psbx","group_id":null}]},{"name":"Backend infrastructure upgrade","status":"completed","created_at":"2026-03-22T07:55:42.943Z","updated_at":"2026-03-22T08:10:07.260Z","monitoring_at":null,"resolved_at":"2026-03-22T08:10:07.243Z","impact":"maintenance","shortlink":"https://stspg.io/38fwj0rls4bq","scheduled_for":"2026-03-22T08:00:00.000Z","scheduled_until":"2026-03-22T08:10:00.000Z","scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":null,"scheduled_auto_in_progress":true,"scheduled_auto_completed":true,"metadata":{},"started_at":"2026-03-22T07:55:42.937Z","reminder_intervals":null,"id":"w445kgsx1dg3","page_id":"chfw0vm4psbx","incident_updates":[{"status":"completed","body":"The scheduled maintenance has been completed.","created_at":"2026-03-22T08:10:07.243Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-03-22T08:10:07.243Z","display_at":"2026-03-22T08:10:07.243Z","deliver_notifications":true,"tweet_id":null,"id":"frtcm5rb7lzk","incident_id":"w445kgsx1dg3","custom_tweet":null,"affected_components":[{"code":"vwbvg0ccxq2q","name":"Storage","old_status":"under_maintenance","new_status":"operational"},{"code":"sgbqpgt5bvl2","name":"Video on Demand","old_status":"under_maintenance","new_status":"operational"},{"code":"wb2bk2wx03y3","name":"Live Scheduler","old_status":"under_maintenance","new_status":"operational"}]},{"status":"in_progress","body":"The scheduled database infrastructure update is in progress.  We will provide updates as necessary.","created_at":"2026-03-22T08:01:29.331Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-03-22T08:01:29.331Z","display_at":"2026-03-22T08:01:29.331Z","deliver_notifications":true,"tweet_id":null,"id":"98936jttgg9n","incident_id":"w445kgsx1dg3","custom_tweet":null,"affected_components":[{"code":"vwbvg0ccxq2q","name":"Storage","old_status":"under_maintenance","new_status":"under_maintenance"},{"code":"sgbqpgt5bvl2","name":"Video on Demand","old_status":"under_maintenance","new_status":"under_maintenance"},{"code":"wb2bk2wx03y3","name":"Live Scheduler","old_status":"under_maintenance","new_status":"under_maintenance"}]},{"status":"in_progress","body":"Scheduled maintenance is currently in progress. We will provide updates as necessary.","created_at":"2026-03-22T08:00:07.339Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-03-22T08:00:07.339Z","display_at":"2026-03-22T08:00:07.339Z","deliver_notifications":true,"tweet_id":null,"id":"f71spn639z60","incident_id":"w445kgsx1dg3","custom_tweet":null,"affected_components":[{"code":"vwbvg0ccxq2q","name":"Storage","old_status":"operational","new_status":"under_maintenance"},{"code":"sgbqpgt5bvl2","name":"Video on Demand","old_status":"operational","new_status":"under_maintenance"},{"code":"wb2bk2wx03y3","name":"Live Scheduler","old_status":"operational","new_status":"under_maintenance"}]},{"status":"scheduled","body":"A backend infrastructure upgrade is planned to improve the performance and long-term stability of our services.  The update will take approximately 10 minutes to action and system availability should be back to normal within 30 minutes if the changes are successful.\n\nDuring the update window the following aspects of the product will be affected so customers should avoid changes in these product areas:\nLive Scheduler: Users may experience errors or failures when attempting to add, update, or delete live schedules.\nChannels: Users may experience errors or failures when attempting to add, update, or delete channels or playlists","created_at":"2026-03-22T07:55:42.990Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-03-22T07:55:42.990Z","display_at":"2026-03-22T07:55:42.990Z","deliver_notifications":true,"tweet_id":null,"id":"wtbkjfzmqbmv","incident_id":"w445kgsx1dg3","custom_tweet":null,"affected_components":[{"code":"vwbvg0ccxq2q","name":"Storage","old_status":"operational","new_status":"operational"},{"code":"sgbqpgt5bvl2","name":"Video on Demand","old_status":"operational","new_status":"operational"},{"code":"wb2bk2wx03y3","name":"Live Scheduler","old_status":"operational","new_status":"operational"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Video on Demand","created_at":"2022-02-22T05:56:14.890Z","updated_at":"2026-03-22T08:10:07.183Z","position":3,"description":null,"showcase":false,"start_date":"2022-02-22T00:00:00.000Z","id":"sgbqpgt5bvl2","page_id":"chfw0vm4psbx","group_id":null},{"status":"operational","name":"Live Scheduler","created_at":"2024-11-13T04:12:48.172Z","updated_at":"2026-03-22T08:10:07.197Z","position":4,"description":null,"showcase":false,"start_date":"2024-11-13T00:00:00.000Z","id":"wb2bk2wx03y3","page_id":"chfw0vm4psbx","group_id":null},{"status":"operational","name":"Storage","created_at":"2022-02-22T05:55:38.090Z","updated_at":"2026-03-22T08:10:07.211Z","position":6,"description":null,"showcase":false,"start_date":"2022-02-22T00:00:00.000Z","id":"vwbvg0ccxq2q","page_id":"chfw0vm4psbx","group_id":null}]},{"name":"Web Portal Access and other services Issue","status":"resolved","created_at":"2025-06-12T18:48:47.184Z","updated_at":"2025-06-20T05:34:40.021Z","monitoring_at":"2025-06-12T20:45:58.265Z","resolved_at":"2025-06-13T02:04:45.012Z","impact":"none","shortlink":"https://stspg.io/p0wsj4hkkgnf","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"none","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2025-06-12T18:48:47.176Z","reminder_intervals":null,"id":"6bdxmtjn7mn2","page_id":"chfw0vm4psbx","incident_updates":[{"status":"resolved","body":"The incident was resolved at Thursday, 12 June 2025 at 20:45 (UTC). All services have been verified, and are operating as expected. We will continue to monitor to ensure ongoing stability.","created_at":"2025-06-13T02:04:45.012Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-06-13T02:04:45.012Z","display_at":"2025-06-13T02:04:45.012Z","deliver_notifications":false,"tweet_id":null,"id":"mq804l552sqy","incident_id":"6bdxmtjn7mn2","custom_tweet":null,"affected_components":[{"code":"vrbgfn5cjnkg","name":"Video Player","old_status":"operational","new_status":"operational"},{"code":"sgbqpgt5bvl2","name":"Video on Demand","old_status":"operational","new_status":"operational"},{"code":"v15spklppw57","name":"Web Portal","old_status":"operational","new_status":"operational"},{"code":"srd5cfcrm9qb","name":"Streaming Events","old_status":"operational","new_status":"operational"},{"code":"wb2bk2wx03y3","name":"Live Scheduler","old_status":"operational","new_status":"operational"}]},{"status":"monitoring","body":"All services have been fully restored and are operating normally. We will continue to monitor platform metrics to ensure ongoing stability and performance.","created_at":"2025-06-12T21:10:41.726Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-06-12T21:10:41.726Z","display_at":"2025-06-12T21:10:41.726Z","deliver_notifications":true,"tweet_id":null,"id":"5g8w5n6vnm95","incident_id":"6bdxmtjn7mn2","custom_tweet":null,"affected_components":[{"code":"vrbgfn5cjnkg","name":"Video Player","old_status":"operational","new_status":"operational"},{"code":"sgbqpgt5bvl2","name":"Video on Demand","old_status":"operational","new_status":"operational"},{"code":"v15spklppw57","name":"Web Portal","old_status":"degraded_performance","new_status":"operational"},{"code":"srd5cfcrm9qb","name":"Streaming Events","old_status":"operational","new_status":"operational"},{"code":"wb2bk2wx03y3","name":"Live Scheduler","old_status":"operational","new_status":"operational"}]},{"status":"monitoring","body":"Services have been restored for the majority of users, and we are continuing to monitor system performance closely.","created_at":"2025-06-12T20:45:58.265Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-06-12T20:45:58.265Z","display_at":"2025-06-12T20:45:58.265Z","deliver_notifications":true,"tweet_id":null,"id":"c7t73w90f3km","incident_id":"6bdxmtjn7mn2","custom_tweet":null,"affected_components":[{"code":"vrbgfn5cjnkg","name":"Video Player","old_status":"partial_outage","new_status":"operational"},{"code":"sgbqpgt5bvl2","name":"Video on Demand","old_status":"partial_outage","new_status":"operational"},{"code":"v15spklppw57","name":"Web Portal","old_status":"partial_outage","new_status":"degraded_performance"},{"code":"srd5cfcrm9qb","name":"Streaming Events","old_status":"partial_outage","new_status":"operational"},{"code":"wb2bk2wx03y3","name":"Live Scheduler","old_status":"partial_outage","new_status":"operational"}]},{"status":"identified","body":"The root cause of the issue has been identified, and mitigation measures have been applied. While some users may still experience intermittent issues, the Web Portal and related services are gradually being restored.","created_at":"2025-06-12T20:28:00.136Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-06-12T20:28:00.136Z","display_at":"2025-06-12T20:28:00.136Z","deliver_notifications":true,"tweet_id":null,"id":"vn979frf22kp","incident_id":"6bdxmtjn7mn2","custom_tweet":null,"affected_components":[{"code":"vrbgfn5cjnkg","name":"Video Player","old_status":"partial_outage","new_status":"partial_outage"},{"code":"sgbqpgt5bvl2","name":"Video on Demand","old_status":"partial_outage","new_status":"partial_outage"},{"code":"v15spklppw57","name":"Web Portal","old_status":"major_outage","new_status":"partial_outage"},{"code":"srd5cfcrm9qb","name":"Streaming Events","old_status":"partial_outage","new_status":"partial_outage"},{"code":"wb2bk2wx03y3","name":"Live Scheduler","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"We are currently experiencing issues with access to the Web Portal and partial disruptions across several services. Our team is actively investigating the root cause, and we will provide further updates as more information becomes available.","created_at":"2025-06-12T19:11:36.191Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-06-12T19:11:36.191Z","display_at":"2025-06-12T19:11:36.191Z","deliver_notifications":true,"tweet_id":null,"id":"1r4fnmynv16f","incident_id":"6bdxmtjn7mn2","custom_tweet":null,"affected_components":[{"code":"vrbgfn5cjnkg","name":"Video Player","old_status":"operational","new_status":"partial_outage"},{"code":"sgbqpgt5bvl2","name":"Video on Demand","old_status":"operational","new_status":"partial_outage"},{"code":"v15spklppw57","name":"Web Portal","old_status":"partial_outage","new_status":"major_outage"},{"code":"srd5cfcrm9qb","name":"Streaming Events","old_status":"operational","new_status":"partial_outage"},{"code":"wb2bk2wx03y3","name":"Live Scheduler","old_status":"operational","new_status":"partial_outage"}]},{"status":"investigating","body":"We are continuing to investigate this issue.","created_at":"2025-06-12T18:50:01.305Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-06-12T18:50:01.305Z","display_at":"2025-06-12T18:50:01.305Z","deliver_notifications":false,"tweet_id":null,"id":"jkbsp01lk53z","incident_id":"6bdxmtjn7mn2","custom_tweet":null,"affected_components":[{"code":"v15spklppw57","name":"Web Portal","old_status":"operational","new_status":"partial_outage"}]},{"status":"investigating","body":"We are aware of an access issue affecting the Web Portal and are currently investigating the cause. We will provide updates as more information becomes available.","created_at":"2025-06-12T18:48:47.252Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-06-12T18:48:47.252Z","display_at":"2025-06-12T18:48:47.252Z","deliver_notifications":true,"tweet_id":null,"id":"0k1lv5zd15l8","incident_id":"6bdxmtjn7mn2","custom_tweet":null,"affected_components":[{"code":"v15spklppw57","name":"Web Portal","old_status":"operational","new_status":"operational"}]}],"postmortem_body":"**Incident Impact and Summary:**\n\nStreamShark alerting and monitoring systems, and numerous end-users reported issues logging into the StreamShark portal at [https://app.streamshark.io](https://app.streamshark.io/) and accessing StreamShark services. The outage extended to other services and end-points at StreamShark, including VoD and Event rendering \\(if they were not already cached\\).\n\nThe issue was caused by a major global outage from one of our primary Cloud infrastructure vendors \\(Google Cloud\\) which impacted a significant number of Google hosted services, and all Google Cloud Availability Zones \\(AZ\\). The relevant Google Cloud incident can be found [here](https://status.cloud.google.com/incidents/ow5i3PPK96RduMcb1SsW), which now includes a full incident report with a root cause analysis. Google summarises the function impacted system below:\n\n\u003e Google and Google Cloud APIs are served through our Google API management and control planes. Distributed regionally, these management and control planes are responsible for ensuring each API request that comes in is authorized, has the policy and appropriate checks \\(like quota\\) to meet their endpoints. The core binary that is part of this policy check system is known as Service Control. Service Control is a regional service that has a regional datastore that it reads quota and policy information from. This datastore metadata gets replicated almost instantly globally to manage quota policies for Google Cloud and our customers.\n\nPer Google’s incident report, Google pushed an insufficiently tested and faulty change to the above mentioned internal Google system, which was subsequently replicated globally, causing widespread service failure across Google hosted services, without appropriate rollback controls. As a result, service recovery took significantly longer than anticipated.\n\n**Impacted components:**\n\n* StreamShark Admin Portal\n* StreamShark API/Backend\n* All StreamShark service end-points \\(VoD, Event, Channel, etc\\) where the end-point was not already cached\n\n**Overview:**\n\nGoogle Cloud is one of multiple Cloud Infrastructure providers used by StreamShark, but a significant one. StreamShark operates across multiple Google Cloud Availability Zones depending on the service and infrastructure needed, per industry standard best practice.\n\nThe Google Cloud issue impacted the majority of Google Cloud services across all available Google Cloud regions. Many StreamShark components were impacted for different durations - the times below encompass the widest possible period of impact.\n\n**Detection:**\n\nDetected by End-user reports and StreamShark automated alerting. The [StreamShark status page](https://status.streamshark.io/) was updated during the issue.\n\n* **First Detected:** Thursday, 12 June 2025 at 18:20 \\(UTC\\)\n* **Mitigated:** Thursday, 12 June 2025 at 20:28 \\(UTC\\)\n* **Ongoing Monitoring:** Thursday, 12 June 2025 at 20:45 \\(UTC\\)\n\n**Remediation:**\n\nThe extent of the incident, simultaneously across multiple services and availability zones was unprecedented - which meant there were minimum levers at our disposal in terms of mitigating the issue in real time. The issue related to an internal quota and policy system in Google Cloud, of which we have no control or visibility on. Google Cloud’s Enterprise Support portal was unavailable during the impacted period so we could not raise a ticket via our Enterprise Support arrangement with Google. However given the scale and global impact of this outage across multiple customers and industries, it would have not expedited any resumption of service.\n\nIn Google’s Incident Report, they have outlined a number of steps they will take to avoid this type of issue from occurring again. Google's response is below:\n\n\u003e **What’s our approach moving forward?**\n\u003e\n\u003e Beyond freezing the system as mentioned above, we will prioritize and safely complete the following:\n\u003e\n\u003e We will modularize Service Control’s architecture, so the functionality is isolated and fails open. Thus, if a corresponding check fails, Service Control can still serve API requests.\n\u003e\n\u003e We will audit all systems that consume globally replicated data. Regardless of the business need for near instantaneous consistency of the data globally \\(i.e. quota management settings are global\\), data replication needs to be propagated incrementally with sufficient time to validate and detect issues.\n\u003e\n\u003e We will enforce all changes to critical binaries to be feature flag protected and disabled by default.\n\u003e\n\u003e We will improve our static analysis and testing practices to correctly handle errors and if need be fail open.\n\u003e\n\u003e We will audit and ensure our systems employ randomized exponential backoff.\n\u003e\n\u003e We will improve our external communications, both automated and human, so our customers get the information they need asap to react to issues, manage their systems and help their customers.\n\u003e\n\u003e We'll ensure our monitoring and communication infrastructure remains operational to serve customers even when Google Cloud and our primary monitoring products are down, ensuring business continuity.\n\n \n\nWhilst we are multi availability and multi vendor across many services, some key services are still hosted on single global Cloud Vendor. We will conduct a feasibility analysis of running more essential StreamShark services across multiple cloud vendors \\(in a hot/cold failover model, or a dual master model\\) in the near future.","postmortem_body_last_updated_at":"2025-06-20T05:34:39.981Z","postmortem_ignored":false,"postmortem_published_at":"2025-06-20T05:34:39.981Z","postmortem_notified_subscribers":true,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Web Portal","created_at":"2024-11-11T04:17:29.199Z","updated_at":"2025-06-12T21:10:41.692Z","position":1,"description":null,"showcase":false,"start_date":"2024-11-11T00:00:00.000Z","id":"v15spklppw57","page_id":"chfw0vm4psbx","group_id":null},{"status":"operational","name":"Streaming Events","created_at":"2024-11-13T04:03:15.137Z","updated_at":"2025-06-12T20:45:58.186Z","position":2,"description":null,"showcase":false,"start_date":"2024-11-13T00:00:00.000Z","id":"srd5cfcrm9qb","page_id":"chfw0vm4psbx","group_id":null},{"status":"operational","name":"Video on Demand","created_at":"2022-02-22T05:56:14.890Z","updated_at":"2026-03-22T08:10:07.183Z","position":3,"description":null,"showcase":false,"start_date":"2022-02-22T00:00:00.000Z","id":"sgbqpgt5bvl2","page_id":"chfw0vm4psbx","group_id":null},{"status":"operational","name":"Live Scheduler","created_at":"2024-11-13T04:12:48.172Z","updated_at":"2026-03-22T08:10:07.197Z","position":4,"description":null,"showcase":false,"start_date":"2024-11-13T00:00:00.000Z","id":"wb2bk2wx03y3","page_id":"chfw0vm4psbx","group_id":null},{"status":"operational","name":"Video Player","created_at":"2022-02-22T05:54:38.442Z","updated_at":"2026-05-21T07:01:49.221Z","position":5,"description":null,"showcase":false,"start_date":"2022-02-22T00:00:00.000Z","id":"vrbgfn5cjnkg","page_id":"chfw0vm4psbx","group_id":null}]},{"name":"Portal Access Issue","status":"resolved","created_at":"2024-11-11T00:43:33.000Z","updated_at":"2024-11-13T03:52:20.571Z","monitoring_at":null,"resolved_at":"2024-11-11T05:52:17.000Z","impact":"minor","shortlink":"https://stspg.io/rm02bbph6f84","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"minor","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2024-11-11T00:43:33.000Z","reminder_intervals":null,"id":"dw60k61nfthf","page_id":"chfw0vm4psbx","incident_updates":[{"status":"resolved","body":"The issue causing errors accessing the Portal has been fully resolved as of 05:52am 11/11/2024 UTC.  We thank you for your patience while we worked on resolving the issue.","created_at":"2024-11-11T06:59:17.941Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-11-11T07:53:17.104Z","display_at":"2024-11-11T05:52:17.000Z","deliver_notifications":true,"tweet_id":null,"id":"lk89jft4v3zs","incident_id":"dw60k61nfthf","custom_tweet":null,"affected_components":[{"code":"vrbgfn5cjnkg","name":"PLAYER","old_status":"operational","new_status":"operational"},{"code":"vwbvg0ccxq2q","name":"STORAGE LAYER","old_status":"operational","new_status":"operational"},{"code":"sgbqpgt5bvl2","name":"VOD","old_status":"operational","new_status":"operational"}]},{"status":"identified","body":"The issue cause has been identified and mitigation has been applied to restore portal access.","created_at":"2024-11-11T04:12:27.172Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-11-11T07:17:14.516Z","display_at":"2024-11-11T02:39:27.000Z","deliver_notifications":true,"tweet_id":null,"id":"kkzgpshsqxdm","incident_id":"dw60k61nfthf","custom_tweet":null,"affected_components":[{"code":"vrbgfn5cjnkg","name":"PLAYER","old_status":"partial_outage","new_status":"operational"},{"code":"vwbvg0ccxq2q","name":"STORAGE LAYER","old_status":"partial_outage","new_status":"operational"},{"code":"sgbqpgt5bvl2","name":"VOD","old_status":"partial_outage","new_status":"operational"}]},{"status":"investigating","body":"We are seeing elevated error rates for requests affecting portal login and a few other service components.  We are currently investigating with our backend vendor to determine root cause.  Further updates to follow.","created_at":"2024-11-11T00:43:33.585Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-11-11T07:07:13.672Z","display_at":"2024-11-11T00:43:33.000Z","deliver_notifications":true,"tweet_id":null,"id":"bp8bn46jvp84","incident_id":"dw60k61nfthf","custom_tweet":null,"affected_components":[{"code":"vrbgfn5cjnkg","name":"PLAYER","old_status":"operational","new_status":"partial_outage"},{"code":"vwbvg0ccxq2q","name":"STORAGE LAYER","old_status":"operational","new_status":"partial_outage"},{"code":"sgbqpgt5bvl2","name":"VOD","old_status":"operational","new_status":"partial_outage"}]}],"postmortem_body":"## Post Mortem - StreamShark Portal Access Issues\n\n### **Summary**\n\nOn Monday, November 11, 2024, at 00:43 UTC, users began experiencing elevated error rates with the StreamShark management portal login function, preventing some users from accessing the portal. The issue was identified and mitigated by 02:39 UTC, restoring access to the portal, and fully resolved at 05:52 am UTC. We sincerely apologize to our customers who were affected during this time and regret any inconvenience this may have caused.\n\n### **Customer Impact:**\n\nBefore November 11, 2024, we received a small number of reports regarding one-off portal login failures, most of which were resolved upon refreshing the page.\n\nFrom November 11, 2024, 00:43 to 02:39 UTC \\(1 hour, 56 minutes\\):\n\n* **Portal Access**: Users may have experienced difficulty logging into the portal via both standard login and SSO login methods.\n* **VoD Deployment**: New Video on Demand \\(VoD\\) deployment was delayed for some users.\n* **Encoder Connection**: Initial connection/publish attempts from both software encoders \\(e.g. OBS, vMix\\) and hardware encoders \\(e.g. Elemental, Pearl\\) may have been incorrectly rejected.\n\nAll other services were unaffected, in particular\n\n* **Player Consumption**: Playback across all platforms \\(Streaming Events, Video on Demand, Live Schedule\\) was unaffected, with continuous uninterrupted service.\n* **Portal functionality**: Once logged into the portal, all standard functions operated as expected.\n\n### **Root cause:**\n\nThe issue originated from a server-side function deployed within a managed cloud environment that began returning inconsistent results when extracting IP addresses for incoming requests.\n\nThe cause of the inconsistency was a recent change in the cloud vendor’s environment which affected the function result. In a subset of cloud instances, instead of returning the expected public IP address, the function incorrectly returned ‘127.0.0.1’ \\(localhost\\). Initially, this staged rollout by the cloud vendor affected only a very small number of users, but the impact escalated on November 11, when a wider number of instances were updated.\n\nThis error resulted in failed authentication attempts impacting login and other relevant functionalities where accurate IP addresses \\(combined with other authentication tokens\\) are essential for security whitelist requirements.\n\n### **Corrective Actions:**\n\n* **Function Update**: The IP extraction function was re-engineered to handle the inconsistent behavior observed from the cloud provider, ensuring accurate IP address retrieval.\n* **Code Audit:** All functionalities dependent on IP accuracy were reviewed and updated to accommodate potential inconsistencies, preventing delays in future service deployment.\n* **Enhanced Monitoring**: New alerting mechanisms were implemented to monitor authentication requests failures related to IP validation. These alerts have been integrated into the standard on-call alerting system to proactively detect and address similar issues in the future.","postmortem_body_last_updated_at":"2024-11-13T03:52:20.553Z","postmortem_ignored":false,"postmortem_published_at":"2024-11-13T03:52:20.553Z","postmortem_notified_subscribers":true,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Video on Demand","created_at":"2022-02-22T05:56:14.890Z","updated_at":"2026-03-22T08:10:07.183Z","position":3,"description":null,"showcase":false,"start_date":"2022-02-22T00:00:00.000Z","id":"sgbqpgt5bvl2","page_id":"chfw0vm4psbx","group_id":null},{"status":"operational","name":"Video Player","created_at":"2022-02-22T05:54:38.442Z","updated_at":"2026-05-21T07:01:49.221Z","position":5,"description":null,"showcase":false,"start_date":"2022-02-22T00:00:00.000Z","id":"vrbgfn5cjnkg","page_id":"chfw0vm4psbx","group_id":null},{"status":"operational","name":"Storage","created_at":"2022-02-22T05:55:38.090Z","updated_at":"2026-03-22T08:10:07.211Z","position":6,"description":null,"showcase":false,"start_date":"2022-02-22T00:00:00.000Z","id":"vwbvg0ccxq2q","page_id":"chfw0vm4psbx","group_id":null}]},{"name":"gce-k-eu-west-2.streamshark.io scheduled update","status":"completed","created_at":"2023-01-09T04:21:24.615Z","updated_at":"2023-01-09T05:38:35.165Z","monitoring_at":null,"resolved_at":"2023-01-09T05:38:35.143Z","impact":"maintenance","shortlink":"https://stspg.io/mbp9nhxp51tf","scheduled_for":"2023-01-09T05:00:00.000Z","scheduled_until":"2023-01-09T05:30:00.000Z","scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":null,"scheduled_auto_in_progress":true,"scheduled_auto_completed":false,"metadata":{},"started_at":"2023-01-09T04:21:24.610Z","reminder_intervals":null,"id":"brrjy28m4k9h","page_id":"chfw0vm4psbx","incident_updates":[{"status":"completed","body":"The scheduled maintenance has been completed.","created_at":"2023-01-09T05:38:35.143Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-01-09T05:38:35.143Z","display_at":"2023-01-09T05:38:35.143Z","deliver_notifications":true,"tweet_id":null,"id":"x3f6n6j2151d","incident_id":"brrjy28m4k9h","custom_tweet":null,"affected_components":[{"code":"vrbgfn5cjnkg","name":"PLAYER","old_status":"under_maintenance","new_status":"operational"}]},{"status":"in_progress","body":"Scheduled maintenance is currently in progress. We will provide updates as necessary.","created_at":"2023-01-09T05:00:52.769Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-01-09T05:00:52.769Z","display_at":"2023-01-09T05:00:52.769Z","deliver_notifications":true,"tweet_id":null,"id":"db8fglj3dlnn","incident_id":"brrjy28m4k9h","custom_tweet":null,"affected_components":[{"code":"vrbgfn5cjnkg","name":"PLAYER","old_status":"operational","new_status":"under_maintenance"}]},{"status":"scheduled","body":"Shared ingest server gce-k-eu-west-2.streamshark.io will have updates applied.  The server will be unavailable for a maximum of 30 minutes.","created_at":"2023-01-09T04:21:24.662Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-01-09T04:21:24.662Z","display_at":"2023-01-09T04:21:24.662Z","deliver_notifications":true,"tweet_id":null,"id":"d2nppt1t611n","incident_id":"brrjy28m4k9h","custom_tweet":null,"affected_components":[{"code":"vrbgfn5cjnkg","name":"PLAYER","old_status":"operational","new_status":"operational"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Video Player","created_at":"2022-02-22T05:54:38.442Z","updated_at":"2026-05-21T07:01:49.221Z","position":5,"description":null,"showcase":false,"start_date":"2022-02-22T00:00:00.000Z","id":"vrbgfn5cjnkg","page_id":"chfw0vm4psbx","group_id":null}]},{"name":"Application database performance degraded","status":"resolved","created_at":"2022-11-15T16:50:57.535Z","updated_at":"2022-11-15T22:30:51.257Z","monitoring_at":"2022-11-15T18:09:00.849Z","resolved_at":"2022-11-15T19:07:07.000Z","impact":"minor","shortlink":"https://stspg.io/78brb7sn9m9z","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"minor","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2022-11-15T16:50:57.529Z","reminder_intervals":null,"id":"fgqpt72zdcl8","page_id":"chfw0vm4psbx","incident_updates":[{"status":"resolved","body":"Our upstream DB provider has advised as of 18:40 UTC, the database issue has been resolved.","created_at":"2022-11-15T19:07:07.836Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2022-11-15T22:30:51.241Z","display_at":"2022-11-15T19:07:07.000Z","deliver_notifications":true,"tweet_id":null,"id":"4wgvtx86vjjp","incident_id":"fgqpt72zdcl8","custom_tweet":null,"affected_components":[{"code":"vwbvg0ccxq2q","name":"STORAGE LAYER","old_status":"degraded_performance","new_status":"operational"}]},{"status":"monitoring","body":"We have attempted mitigation steps and have escalated the issue to the upstream database cloud host, who concurrently acknowledged the issue, and is actively rectifying it. Error rates have dropped significantly and function appears to be restored. We will continue to monitor the issue and report when it's fully resolved here.","created_at":"2022-11-15T18:09:00.849Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2022-11-15T18:09:00.849Z","display_at":"2022-11-15T18:09:00.849Z","deliver_notifications":true,"tweet_id":null,"id":"r8b5xq09q3qp","incident_id":"fgqpt72zdcl8","custom_tweet":null,"affected_components":[{"code":"vwbvg0ccxq2q","name":"STORAGE LAYER","old_status":"degraded_performance","new_status":"degraded_performance"}]},{"status":"investigating","body":"An intermittent DB issue is causing elevated error rates - we are currently working with our DB provider to determine and address the root cause","created_at":"2022-11-15T16:50:57.621Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2022-11-15T16:50:57.621Z","display_at":"2022-11-15T16:50:57.621Z","deliver_notifications":true,"tweet_id":null,"id":"tx69ts449ys1","incident_id":"fgqpt72zdcl8","custom_tweet":null,"affected_components":[{"code":"vwbvg0ccxq2q","name":"STORAGE LAYER","old_status":"operational","new_status":"degraded_performance"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Storage","created_at":"2022-02-22T05:55:38.090Z","updated_at":"2026-03-22T08:10:07.211Z","position":6,"description":null,"showcase":false,"start_date":"2022-02-22T00:00:00.000Z","id":"vwbvg0ccxq2q","page_id":"chfw0vm4psbx","group_id":null}]},{"name":"Events can't update phase","status":"resolved","created_at":"2022-05-17T04:12:40.622Z","updated_at":"2022-05-18T00:58:25.754Z","monitoring_at":null,"resolved_at":"2022-05-18T00:58:25.741Z","impact":"major","shortlink":"https://stspg.io/gsmll6yjhgsm","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":null,"scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2022-05-17T04:12:40.618Z","reminder_intervals":null,"id":"58h8l0rqp5qv","page_id":"chfw0vm4psbx","incident_updates":[{"status":"resolved","body":"We have found the root cause of the issue and a patch has been applied","created_at":"2022-05-18T00:58:25.741Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2022-05-18T00:58:25.741Z","display_at":"2022-05-18T00:58:25.741Z","deliver_notifications":true,"tweet_id":null,"id":"2vcc2895h6xm","incident_id":"58h8l0rqp5qv","custom_tweet":null,"affected_components":[{"code":"vwbvg0ccxq2q","name":"STORAGE LAYER","old_status":"partial_outage","new_status":"operational"}]},{"status":"identified","body":"The issue has been identified and a fix is being implemented.","created_at":"2022-05-17T04:12:40.686Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2022-05-17T04:12:40.686Z","display_at":"2022-05-17T04:12:40.686Z","deliver_notifications":true,"tweet_id":null,"id":"mj35gcjc6x5k","incident_id":"58h8l0rqp5qv","custom_tweet":null,"affected_components":[{"code":"vwbvg0ccxq2q","name":"STORAGE LAYER","old_status":"operational","new_status":"partial_outage"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Storage","created_at":"2022-02-22T05:55:38.090Z","updated_at":"2026-03-22T08:10:07.211Z","position":6,"description":null,"showcase":false,"start_date":"2022-02-22T00:00:00.000Z","id":"vwbvg0ccxq2q","page_id":"chfw0vm4psbx","group_id":null}]}]
}