{ "protocolSetVersion": "gm-ai-consumer-compat/4.0.0", "bundleId": "ae798d6a-9f73-4413-9524-22297cbf7de0", "scanId": "806017f3-18f7-475d-ae74-d0612c54dcf3", "targetUrl": "https://www.usa.gov/", "auditProfile": { "profileId": "public-ai-discovery", "profileVersion": "1.0.0", "displayName": "Public AI discovery compatibility", "actorRequirements": { "openai:oai-searchbot": "REQUIRED", "anthropic:claude-searchbot": "REQUIRED", "google:googlebot": "REQUIRED", "perplexity:perplexitybot": "REQUIRED", "openai:gptbot": "INFORMATIONAL", "anthropic:claudebot": "INFORMATIONAL", "google:google-extended": "INFORMATIONAL", "openai:chatgpt-user": "INFORMATIONAL", "anthropic:claude-user": "INFORMATIONAL", "perplexity:perplexity-user": "INFORMATIONAL", "google:google-agent": "INFORMATIONAL", "openai:oai-adsbot": "INFORMATIONAL", "kagi:kagibot": "INFORMATIONAL", "commoncrawl:ccbot": "INFORMATIONAL", "microsoft:bingbot": "INFORMATIONAL", "apple:applebot": "INFORMATIONAL", "apple:applebot-extended": "INFORMATIONAL", "duckduckgo:duckassistbot": "INFORMATIONAL", "diffbot:diffbot": "INFORMATIONAL", "webzio:omgili": "INFORMATIONAL", "allenai:ai2bot": "INFORMATIONAL", "hive:imagesiftbot": "INFORMATIONAL", "linkup:linkupbot": "INFORMATIONAL", "google:google-cloudvertexbot": "INFORMATIONAL", "google:google-gemininotebook": "INFORMATIONAL", "google:googleother": "INFORMATIONAL", "firecrawl:firecrawlagent": "INFORMATIONAL" }, "limitations": [ "This profile tests documented public discovery controls; it does not prove indexing, retrieval, ranking, citation, training inclusion, or model use.", "User-initiated agents are informational because provider behavior may be context dependent.", "Informational actors never create blockers and are never required; an unmeasured informational actor is reported rather than inferred." ] }, "createdAt": "2026-10-23T12:23:12.627Z", "artifacts": [ { "artifactId": "f0cb1e46-f740-49c5-8cbb-ab49da5046db", "kind": "RAW_HTTP_RESPONSE", "representation": "RAW_HTTP", "requestedUrl": "https://www.usa.gov/", "finalUrl": "https://www.usa.gov/", "capturedAt": "2026-10-23T12:23:13.857Z", "requestMethod": "GET", "requestUserAgent": "GenerativeMetrics/1.0 (+https://generativemetrics.com)", "httpStatus": 200, "contentType": "text/html", "responseHeaders": { "age": "124", "cache-control": "max-age=900", "content-encoding": "br", "content-type": "text/html", "etag": "W/\"f8488ef7bbb8c326e90563b3b8694732\"", "last-modified": "Thu, 22 Oct 2026 19:46:15 GMT", "vary": "Accept-Encoding,Accept-Encoding,Accept-Encoding" }, "redactedHeaderNames": [ "connection", "date", "strict-transport-security", "transfer-encoding", "via", "x-amz-cf-id", "x-amz-cf-pop", "x-cache", "x-content-type-options", "x-frame-options", "x-vcap-request-id" ], "byteLength": 48647, "sha256": "34f9e0bd13c46c13ffa29001081090b570897d5bdf045bad9aebe3823d937ece", "sha256Scope": "FULL", "bodyTruncated": false, "redirects": [], "retention": "HASH_ONLY", "storageRef": null, "limitations": [] }, { "artifactId": "69a414b7-f69c-44f6-8c02-289242626038", "kind": "ROBOTS_TXT", "representation": "POLICY_DOCUMENT", "requestedUrl": "https://www.usa.gov/robots.txt", "finalUrl": "https://www.usa.gov/robots.txt", "capturedAt": "2026-10-23T12:23:14.600Z", "requestMethod": "GET", "requestUserAgent": "GenerativeMetrics/1.0 (+https://generativemetrics.com)", "httpStatus": 200, "contentType": "text/plain", "responseHeaders": { "age": "461", "cache-control": "max-age=900", "content-encoding": "br", "content-type": "text/plain", "etag": "W/\"071343f30a545bf62cfbb6c19794c187\"", "last-modified": "Thu, 22 Oct 2026 19:20:26 GMT", "vary": "Accept-Encoding,Accept-Encoding,Accept-Encoding" }, "redactedHeaderNames": [ "connection", "date", "strict-transport-security", "transfer-encoding", "via", "x-amz-cf-id", "x-amz-cf-pop", "x-cache", "x-content-type-options", "x-frame-options", "x-vcap-request-id" ], "byteLength": 1558, "sha256": "57210da4611cc41b25fb10231cdadd60520cb5ef20ad92309cdd0df6fd12377a", "sha256Scope": "FULL", "bodyTruncated": false, "redirects": [], "retention": "HASH_ONLY", "storageRef": null, "limitations": [] }, { "artifactId": "f3313a08-aa84-4148-b6c7-c54fcb071c59", "kind": "LLMS_TXT", "representation": "POLICY_DOCUMENT", "requestedUrl": "https://www.usa.gov/llms.txt", "finalUrl": "https://www.usa.gov/llms.txt", "capturedAt": "2026-10-23T12:23:15.931Z", "requestMethod": "GET", "requestUserAgent": "GenerativeMetrics/1.0 (+https://generativemetrics.com)", "httpStatus": 404, "reasonCode": "NOT_FOUND", "contentType": "text/html", "responseHeaders": { "content-encoding": "br", "content-type": "text/html", "etag": "W/\"a2ed3c678a90f1f54eb8b9fae747068d\"", "last-modified": "Thu, 22 Oct 2026 19:46:15 GMT", "vary": "Accept-Encoding,Accept-Encoding,Accept-Encoding" }, "redactedHeaderNames": [ "connection", "date", "strict-transport-security", "transfer-encoding", "via", "x-amz-cf-id", "x-amz-cf-pop", "x-cache", "x-content-type-options", "x-frame-options", "x-vcap-request-id" ], "byteLength": 0, "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "sha256Scope": "FULL", "bodyTruncated": false, "redirects": [], "retention": "HASH_ONLY", "storageRef": null, "limitations": [ "HTTP 404: the resource was not received, so this artifact records the response status (NOT_FOUND) — the body served with the status is not sealed as resource content." ] }, { "artifactId": "3febc7be-434b-4720-889e-7f04cb6d7b2b", "kind": "SITEMAP_XML", "representation": "POLICY_DOCUMENT", "requestedUrl": "https://www.usa.gov/sitemap.xml", "finalUrl": "https://www.usa.gov/sitemap.xml", "capturedAt": "2026-10-23T12:23:16.918Z", "requestMethod": "GET", "requestUserAgent": "GenerativeMetrics/1.0 (+https://generativemetrics.com)", "httpStatus": 200, "contentType": "text/xml", "responseHeaders": { "cache-control": "max-age=900", "content-encoding": "br", "content-type": "text/xml", "etag": "W/\"3eaa113c568984d6acdb80889e85523e\"", "last-modified": "Thu, 22 Oct 2026 19:46:16 GMT", "vary": "Accept-Encoding,Accept-Encoding,Accept-Encoding" }, "redactedHeaderNames": [ "connection", "date", "strict-transport-security", "transfer-encoding", "via", "x-amz-cf-id", "x-amz-cf-pop", "x-cache", "x-content-type-options", "x-frame-options", "x-vcap-request-id" ], "byteLength": 124444, "sha256": "92a01edf234586d0654052387d61d8f290521555216cdb756654557d6916da41", "sha256Scope": "FULL", "bodyTruncated": false, "redirects": [], "retention": "HASH_ONLY", "storageRef": null, "limitations": [] }, { "artifactId": "b13cb7cf-413b-44b1-8e91-d561d830fc31", "kind": "RAW_HTTP_RESPONSE", "representation": "RAW_HTTP", "requestedUrl": "https://www.usa.gov/", "finalUrl": "https://www.usa.gov/", "capturedAt": "2026-10-23T12:23:17.645Z", "requestMethod": "GET", "requestUserAgent": "GenerativeMetrics/1.0 (+https://generativemetrics.com)", "httpStatus": 200, "contentType": "text/html", "responseHeaders": { "age": "128", "cache-control": "max-age=900", "content-encoding": "br", "content-type": "text/html", "etag": "W/\"f8488ef7bbb8c326e90563b3b8694732\"", "last-modified": "Thu, 22 Oct 2026 19:46:15 GMT", "vary": "Accept-Encoding,Accept-Encoding,Accept-Encoding" }, "redactedHeaderNames": [ "connection", "date", "strict-transport-security", "transfer-encoding", "via", "x-amz-cf-id", "x-amz-cf-pop", "x-cache", "x-content-type-options", "x-frame-options", "x-vcap-request-id" ], "byteLength": 48647, "sha256": "34f9e0bd13c46c13ffa29001081090b570897d5bdf045bad9aebe3823d937ece", "sha256Scope": "FULL", "bodyTruncated": false, "redirects": [], "retention": "HASH_ONLY", "storageRef": null, "limitations": [] }, { "artifactId": "554ed831-6d27-4c75-83a7-0eace8fc2eb8", "kind": "RENDERED_DOM", "representation": "RENDERED_BROWSER", "requestedUrl": "https://www.usa.gov/", "finalUrl": "https://www.usa.gov/", "capturedAt": "2026-10-23T12:22:57.172Z", "requestMethod": "GET", "requestUserAgent": null, "httpStatus": null, "contentType": "text/html", "responseHeaders": {}, "redactedHeaderNames": [], "byteLength": 54162, "sha256": "14646fb1285ea7e61769151229bc39497977ea07262fdd0861cbb98f4b944c3c", "sha256Scope": "FULL", "bodyTruncated": false, "redirects": [], "retention": "HASH_ONLY", "storageRef": null, "limitations": [ "Browser-derived artifact; HTTP status and response headers belong to the raw HTTP acquisition artifact." ] }, { "artifactId": "8c624ab2-9f36-4a02-a4e2-43f3d2fd5b9b", "kind": "NO_JAVASCRIPT_DOM", "representation": "NO_JAVASCRIPT_BROWSER", "requestedUrl": "https://www.usa.gov/", "finalUrl": "https://www.usa.gov/", "capturedAt": "2026-10-23T12:22:59.824Z", "requestMethod": "GET", "requestUserAgent": null, "httpStatus": null, "contentType": "text/html", "responseHeaders": {}, "redactedHeaderNames": [], "byteLength": 48705, "sha256": "1d9a045f7b32cdfb606d91e735a75885c6ccf2610ab03e709a0961efc0e37365", "sha256Scope": "FULL", "bodyTruncated": false, "redirects": [], "retention": "HASH_ONLY", "storageRef": null, "limitations": [ "Browser-derived artifact; HTTP status and response headers belong to the raw HTTP acquisition artifact." ] } ], "protocolExecutions": [ { "protocolExecutionId": "cb943ad1-bd64-401f-9ebd-833a8f9b456b", "protocolId": "acquisition.http", "protocolVersion": "1.0.0", "stage": "ACQUISITION", "required": true, "actor": { "actorId": "gm:direct-http-probe", "provider": "Generative Metrics", "displayName": "Generative Metrics direct HTTP probe", "purpose": "UNSPECIFIED", "robotsPolicyApplicability": "NOT_APPLICABLE", "userAgent": "GenerativeMetrics/1.0 (+https://generativemetrics.com)" }, "state": "SUCCEEDED", "startedAt": "2026-10-23T12:23:12.627Z", "finishedAt": "2026-10-23T12:23:13.857Z", "artifactRefs": [ "f0cb1e46-f740-49c5-8cbb-ab49da5046db" ], "input": { "requestedUrl": "https://www.usa.gov/", "timeoutMs": 15000, "maxBodyBytes": 2097152, "maxRedirects": 10 }, "observations": [ { "observationId": "29bd2269-fb19-4376-9c03-7fb089bce21c", "protocolExecutionId": "cb943ad1-bd64-401f-9ebd-833a8f9b456b", "kind": "HTTP_RESPONSE", "result": "OBSERVED", "evidenceClass": "DIRECT", "value": { "statusCode": 200, "finalUrl": "https://www.usa.gov/", "redirectCount": 0, "contentType": "text/html", "capturedByteLength": 48647, "bodyTruncated": false, "responseHeaders": { "age": "124", "cache-control": "max-age=900", "content-encoding": "br", "content-type": "text/html", "etag": "W/\"f8488ef7bbb8c326e90563b3b8694732\"", "last-modified": "Thu, 22 Oct 2026 19:46:15 GMT", "vary": "Accept-Encoding,Accept-Encoding,Accept-Encoding" }, "cacheState": { "cache-control": "max-age=900", "age": "124" } }, "artifactRefs": [ "f0cb1e46-f740-49c5-8cbb-ab49da5046db" ], "producedAt": "2026-10-23T12:23:13.857Z", "limitations": [ "Cache-state headers (cache-control, age, cf-cache-status) were observed on this response; they describe this fetch only and do not establish what any other client or fetch would receive." ] } ], "failure": null, "limitations": [] }, { "protocolExecutionId": "031f4eac-2768-4b17-851e-6dd5c771f91c", "protocolId": "acquisition.robots", "protocolVersion": "1.0.0", "stage": "ACQUISITION", "required": true, "actor": { "actorId": "gm:direct-http-probe", "provider": "Generative Metrics", "displayName": "Generative Metrics direct HTTP probe", "purpose": "UNSPECIFIED", "robotsPolicyApplicability": "NOT_APPLICABLE", "userAgent": "GenerativeMetrics/1.0 (+https://generativemetrics.com)" }, "state": "SUCCEEDED", "startedAt": "2026-10-23T12:23:12.628Z", "finishedAt": "2026-10-23T12:23:14.600Z", "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "input": { "requestedUrl": "https://www.usa.gov/robots.txt", "timeoutMs": 15000, "maxBodyBytes": 2097152, "maxRedirects": 10 }, "observations": [ { "observationId": "a85f3de0-6cc8-4763-b0c8-b5f9d28c7d64", "protocolExecutionId": "031f4eac-2768-4b17-851e-6dd5c771f91c", "kind": "ROBOTS_HTTP_RESPONSE", "result": "OBSERVED", "evidenceClass": "DIRECT", "value": { "statusCode": 200, "finalUrl": "https://www.usa.gov/robots.txt", "redirectCount": 0, "contentType": "text/plain", "capturedByteLength": 1558, "bodyTruncated": false, "responseHeaders": { "age": "461", "cache-control": "max-age=900", "content-encoding": "br", "content-type": "text/plain", "etag": "W/\"071343f30a545bf62cfbb6c19794c187\"", "last-modified": "Thu, 22 Oct 2026 19:20:26 GMT", "vary": "Accept-Encoding,Accept-Encoding,Accept-Encoding" }, "cacheState": { "cache-control": "max-age=900", "age": "461" } }, "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "producedAt": "2026-10-23T12:23:14.600Z", "limitations": [ "Cache-state headers (cache-control, age, cf-cache-status) were observed on this response; they describe this fetch only and do not establish what any other client or fetch would receive." ] } ], "failure": null, "limitations": [] }, { "protocolExecutionId": "96be7182-8825-40b5-acf6-c79fc27d46bb", "protocolId": "acquisition.llms-txt", "protocolVersion": "1.0.0", "stage": "ACQUISITION", "required": false, "actor": { "actorId": "gm:llms-txt-probe", "provider": "Generative Metrics", "displayName": "Generative Metrics /llms.txt probe", "purpose": "UNSPECIFIED", "robotsPolicyApplicability": "NOT_APPLICABLE", "userAgent": "GenerativeMetrics/1.0 (+https://generativemetrics.com)" }, "state": "SUCCEEDED", "startedAt": "2026-10-23T12:23:12.628Z", "finishedAt": "2026-10-23T12:23:15.932Z", "artifactRefs": [ "f3313a08-aa84-4148-b6c7-c54fcb071c59" ], "input": { "targetUrl": "https://www.usa.gov/" }, "observations": [ { "observationId": "05c0499e-4df0-48d0-b74b-fa1e57f074ea", "protocolExecutionId": "96be7182-8825-40b5-acf6-c79fc27d46bb", "kind": "LLMS_TXT_STATUS", "result": "NOT_OBSERVED", "evidenceClass": "DIRECT", "value": { "status": 404, "byteLength": 0, "mediaType": "text/html", "present": false, "titleHeadingObserved": null, "valid": null, "truncated": false }, "artifactRefs": [ "f3313a08-aa84-4148-b6c7-c54fcb071c59" ], "producedAt": "2026-10-23T12:23:15.932Z", "limitations": [ "The target does not publish a /llms.txt file." ] } ], "failure": null, "limitations": [] }, { "protocolExecutionId": "3b289399-4448-4493-96ca-59939a5f9bb7", "protocolId": "acquisition.sitemap", "protocolVersion": "1.0.0", "stage": "ACQUISITION", "required": false, "actor": { "actorId": "gm:sitemap-probe", "provider": "Generative Metrics", "displayName": "Generative Metrics /sitemap.xml probe", "purpose": "UNSPECIFIED", "robotsPolicyApplicability": "NOT_APPLICABLE", "userAgent": "GenerativeMetrics/1.0 (+https://generativemetrics.com)" }, "state": "SUCCEEDED", "startedAt": "2026-10-23T12:23:12.629Z", "finishedAt": "2026-10-23T12:23:16.918Z", "artifactRefs": [ "3febc7be-434b-4720-889e-7f04cb6d7b2b" ], "input": { "targetUrl": "https://www.usa.gov/" }, "observations": [ { "observationId": "c81ada46-a616-4e54-8ca7-79205c6c7200", "protocolExecutionId": "3b289399-4448-4493-96ca-59939a5f9bb7", "kind": "SITEMAP_STATUS", "result": "OBSERVED", "evidenceClass": "DIRECT", "value": { "status": 200, "byteLength": 124444, "mediaType": "text/xml", "present": true, "valid": true, "truncated": false, "sitemapIndex": false, "urlCount": 1526, "newestLastmod": null, "targetUrlListed": true }, "artifactRefs": [ "3febc7be-434b-4720-889e-7f04cb6d7b2b" ], "producedAt": "2026-10-23T12:23:16.918Z", "limitations": [] } ], "failure": null, "limitations": [] }, { "protocolExecutionId": "51f482d2-32e9-4bf7-b264-88ca529dda73", "protocolId": "content.negotiation", "protocolVersion": "1.0.0", "stage": "ACQUISITION", "required": false, "actor": { "actorId": "gm:content-negotiation-probe", "provider": "Generative Metrics", "displayName": "Generative Metrics content-negotiation probe", "purpose": "UNSPECIFIED", "robotsPolicyApplicability": "NOT_APPLICABLE", "userAgent": "GenerativeMetrics/1.0 (+https://generativemetrics.com)" }, "startedAt": "2026-10-23T12:23:12.630Z", "finishedAt": "2026-10-23T12:23:17.646Z", "input": { "requestedUrl": "https://www.usa.gov/", "acceptHeader": "text/markdown", "maxBodyBytes": 262144 }, "failure": null, "artifactRefs": [ "b13cb7cf-413b-44b1-8e91-d561d830fc31" ], "state": "SUCCEEDED", "observations": [ { "observationId": "dc74ad6f-2b6c-4dbb-8fb1-3ec06f848f03", "protocolExecutionId": "51f482d2-32e9-4bf7-b264-88ca529dda73", "kind": "CONTENT_NEGOTIATION", "result": "OBSERVED", "evidenceClass": "DIRECT", "value": { "requestedUrl": "https://www.usa.gov/", "acceptHeader": "text/markdown", "status": 200, "mediaType": "text/html", "byteLength": 48647, "truncated": false, "markdownRepresentationServed": false, "htmlRepresentationServed": true, "bodyStartsWithMarkup": true, "advertisedMarkdownAlternate": false, "advertisedMarkdownAlternateTarget": null }, "artifactRefs": [ "b13cb7cf-413b-44b1-8e91-d561d830fc31" ], "producedAt": "2026-10-23T12:23:17.646Z", "limitations": [ "The request asked for `Accept: text/markdown` and the origin answered HTTP 200 with text/html.", "CONTENT_NEGOTIATION_CONVENTION: convention — serving a markdown representation is a publishing convention, not a requirement; this records what was requested and what came back and never infers how a provider consumes content.", "Only the HTTP status, media type, bounded byte length, truncation flag and markup-prefix boolean are recorded; no response body is retained.", "Markdown detection uses the declared media type plus a 512-character body prefix check, so a mislabelled representation is recorded as declared.", "The response body was read with a 262144-byte bound.", "Only a Link header entry with rel=\"alternate\" and a markdown media type is recognised as the origin advertising an alternate markdown representation." ] } ], "limitations": [ "The request asked for `Accept: text/markdown` and the origin answered HTTP 200 with text/html.", "CONTENT_NEGOTIATION_CONVENTION: convention — serving a markdown representation is a publishing convention, not a requirement; this records what was requested and what came back and never infers how a provider consumes content.", "Only the HTTP status, media type, bounded byte length, truncation flag and markup-prefix boolean are recorded; no response body is retained.", "Markdown detection uses the declared media type plus a 512-character body prefix check, so a mislabelled representation is recorded as declared.", "The response body was read with a 262144-byte bound.", "Only a Link header entry with rel=\"alternate\" and a markdown media type is recognised as the origin advertising an alternate markdown representation." ] }, { "protocolExecutionId": "68f82ed6-d852-46fa-8333-8bbbb116a73c", "protocolId": "representation.raw-rendered", "protocolVersion": "1.0.0", "stage": "EXTRACTION", "required": true, "actor": { "actorId": "gm:direct-http-probe", "provider": "Generative Metrics", "displayName": "Generative Metrics direct HTTP probe", "purpose": "UNSPECIFIED", "robotsPolicyApplicability": "NOT_APPLICABLE", "userAgent": "GenerativeMetrics/1.0 (+https://generativemetrics.com)" }, "startedAt": "2026-10-23T12:23:17.648Z", "input": { "rawArtifactId": "f0cb1e46-f740-49c5-8cbb-ab49da5046db", "renderedRepresentationRequested": true, "noJavaScriptRepresentationRequested": true, "renderedCapturedAt": "2026-10-23T12:22:57.172Z", "renderedCaptureTimestampSource": "PLAYWRIGHT_SERVICE", "noJavaScriptCapturedAt": "2026-10-23T12:22:59.824Z", "noJavaScriptCaptureTimestampSource": "PLAYWRIGHT_SERVICE" }, "failure": null, "finishedAt": "2026-10-23T12:23:17.745Z", "state": "SUCCEEDED", "artifactRefs": [ "f0cb1e46-f740-49c5-8cbb-ab49da5046db", "554ed831-6d27-4c75-83a7-0eace8fc2eb8", "8c624ab2-9f36-4a02-a4e2-43f3d2fd5b9b" ], "observations": [ { "observationId": "6b909c3b-05db-45f0-bd9c-ba94b095a800", "protocolExecutionId": "68f82ed6-d852-46fa-8333-8bbbb116a73c", "kind": "REPRESENTATION_BYTE_COMPARISON", "result": "OBSERVED", "evidenceClass": "DERIVED", "value": { "rawHttpBytes": 48647, "renderedBrowserBytes": 54162, "noJavaScriptBrowserBytes": 48705, "rawToRenderedByteRatio": 0.8982, "rawAndRenderedHashesEqual": false, "captureSkewMs": 16685, "noJavaScriptCaptureSkewMs": 14033, "semanticEquivalence": "NOT_EVALUATED" }, "artifactRefs": [ "f0cb1e46-f740-49c5-8cbb-ab49da5046db", "554ed831-6d27-4c75-83a7-0eace8fc2eb8", "8c624ab2-9f36-4a02-a4e2-43f3d2fd5b9b" ], "producedAt": "2026-10-23T12:23:17.745Z", "limitations": [ "Byte and hash comparisons do not establish semantic equivalence.", "Browser-rendered and no-JavaScript artifacts describe Generative Metrics probes, not universal crawler behavior.", "Raw HTTP and browser-rendered artifacts are sequential observations, not the same network response." ] } ], "limitations": [ "Byte and hash comparisons do not establish semantic equivalence.", "Browser-rendered and no-JavaScript artifacts describe Generative Metrics probes, not universal crawler behavior.", "Raw HTTP and browser-rendered artifacts are sequential observations, not the same network response." ] }, { "protocolExecutionId": "ae477e6d-7a2a-4f0b-9cf1-f2482be7bbf0", "protocolId": "policy.robots", "protocolVersion": "1.0.0", "stage": "POLICY", "required": true, "actor": { "actorId": "openai:oai-searchbot", "provider": "OpenAI", "displayName": "OAI-SearchBot", "purpose": "SEARCH", "robotsPolicyApplicability": "APPLIES", "productToken": "OAI-SearchBot", "probeUserAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; OAI-SearchBot/1.0; +https://openai.com/searchbot)", "documentationUrl": "https://developers.openai.com/api/docs/bots", "documentationVerifiedAt": "2026-07-26" }, "startedAt": "2026-10-23T12:23:17.746Z", "finishedAt": "2026-10-23T12:23:17.746Z", "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "input": { "targetUrl": "https://www.usa.gov/", "robotsAcquisitionProtocolExecutionId": "031f4eac-2768-4b17-851e-6dd5c771f91c" }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "95c85f12-972e-4e48-80a1-4c4067d75ed9", "protocolExecutionId": "ae477e6d-7a2a-4f0b-9cf1-f2482be7bbf0", "kind": "ROBOTS_POLICY_DECISION", "result": "NOT_OBSERVED", "evidenceClass": "DERIVED", "value": { "actorId": "openai:oai-searchbot", "actorPurpose": "SEARCH", "applicability": "APPLIES", "decision": "NO_MATCHING_RULE", "targetPath": "/", "matchedUserAgents": [ "*" ], "matchedRule": null, "accessInference": "NOT_ESTABLISHED", "warnings": [], "sitemapDeclarations": { "urls": [ "https://www.usa.gov/sitemap.xml" ], "totalDeclared": 1, "listTruncated": false }, "outsideTokenInventory": { "tokens": [], "totalDistinct": 0, "listTruncated": false, "nonProductTokenCount": 0 }, "declaredUserAgentRuleCount": 0 }, "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "producedAt": "2026-10-23T12:23:17.746Z", "limitations": [ "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] } ], "limitations": [ "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] }, { "protocolExecutionId": "a32b95a1-d580-4368-914f-06826588f0be", "protocolId": "policy.robots", "protocolVersion": "1.0.0", "stage": "POLICY", "required": true, "actor": { "actorId": "anthropic:claude-searchbot", "provider": "Anthropic", "displayName": "Claude-SearchBot", "purpose": "SEARCH", "robotsPolicyApplicability": "APPLIES", "productToken": "Claude-SearchBot", "probeUserAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Claude-SearchBot/1.0; +https://www.anthropic.com/clausearchbot)", "documentationUrl": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "documentationVerifiedAt": "2026-07-26" }, "startedAt": "2026-10-23T12:23:17.748Z", "finishedAt": "2026-10-23T12:23:17.748Z", "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "input": { "targetUrl": "https://www.usa.gov/", "robotsAcquisitionProtocolExecutionId": "031f4eac-2768-4b17-851e-6dd5c771f91c" }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "9b59a00e-69c6-4e9b-a8b3-6e76a52e5267", "protocolExecutionId": "a32b95a1-d580-4368-914f-06826588f0be", "kind": "ROBOTS_POLICY_DECISION", "result": "NOT_OBSERVED", "evidenceClass": "DERIVED", "value": { "actorId": "anthropic:claude-searchbot", "actorPurpose": "SEARCH", "applicability": "APPLIES", "decision": "NO_MATCHING_RULE", "targetPath": "/", "matchedUserAgents": [ "*" ], "matchedRule": null, "accessInference": "NOT_ESTABLISHED", "warnings": [], "sitemapDeclarations": { "urls": [ "https://www.usa.gov/sitemap.xml" ], "totalDeclared": 1, "listTruncated": false }, "outsideTokenInventory": { "tokens": [], "totalDistinct": 0, "listTruncated": false, "nonProductTokenCount": 0 }, "declaredUserAgentRuleCount": 0 }, "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "producedAt": "2026-10-23T12:23:17.748Z", "limitations": [ "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] } ], "limitations": [ "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] }, { "protocolExecutionId": "2e9cd75d-5dbd-4a8e-a77f-86248ff2cc6a", "protocolId": "policy.robots", "protocolVersion": "1.0.0", "stage": "POLICY", "required": true, "actor": { "actorId": "google:googlebot", "provider": "Google", "displayName": "Googlebot", "purpose": "SEARCH", "robotsPolicyApplicability": "APPLIES", "productToken": "Googlebot", "probeUserAgent": "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", "documentationUrl": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "documentationVerifiedAt": "2026-07-26" }, "startedAt": "2026-10-23T12:23:17.749Z", "finishedAt": "2026-10-23T12:23:17.749Z", "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "input": { "targetUrl": "https://www.usa.gov/", "robotsAcquisitionProtocolExecutionId": "031f4eac-2768-4b17-851e-6dd5c771f91c" }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "093f25da-6d9f-44d4-a960-208497b456b1", "protocolExecutionId": "2e9cd75d-5dbd-4a8e-a77f-86248ff2cc6a", "kind": "ROBOTS_POLICY_DECISION", "result": "NOT_OBSERVED", "evidenceClass": "DERIVED", "value": { "actorId": "google:googlebot", "actorPurpose": "SEARCH", "applicability": "APPLIES", "decision": "NO_MATCHING_RULE", "targetPath": "/", "matchedUserAgents": [ "*" ], "matchedRule": null, "accessInference": "NOT_ESTABLISHED", "warnings": [], "sitemapDeclarations": { "urls": [ "https://www.usa.gov/sitemap.xml" ], "totalDeclared": 1, "listTruncated": false }, "outsideTokenInventory": { "tokens": [], "totalDistinct": 0, "listTruncated": false, "nonProductTokenCount": 0 }, "declaredUserAgentRuleCount": 0 }, "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "producedAt": "2026-10-23T12:23:17.749Z", "limitations": [ "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] } ], "limitations": [ "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] }, { "protocolExecutionId": "debfb47a-a66b-43c9-97d2-212ea94793e3", "protocolId": "policy.robots", "protocolVersion": "1.0.0", "stage": "POLICY", "required": true, "actor": { "actorId": "perplexity:perplexitybot", "provider": "Perplexity", "displayName": "PerplexityBot", "purpose": "SEARCH", "robotsPolicyApplicability": "APPLIES", "productToken": "PerplexityBot", "probeUserAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)", "documentationUrl": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers", "documentationVerifiedAt": "2026-07-26" }, "startedAt": "2026-10-23T12:23:17.749Z", "finishedAt": "2026-10-23T12:23:17.749Z", "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "input": { "targetUrl": "https://www.usa.gov/", "robotsAcquisitionProtocolExecutionId": "031f4eac-2768-4b17-851e-6dd5c771f91c" }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "968a7676-c51b-404e-a285-f3b6c27a060a", "protocolExecutionId": "debfb47a-a66b-43c9-97d2-212ea94793e3", "kind": "ROBOTS_POLICY_DECISION", "result": "NOT_OBSERVED", "evidenceClass": "DERIVED", "value": { "actorId": "perplexity:perplexitybot", "actorPurpose": "SEARCH", "applicability": "APPLIES", "decision": "NO_MATCHING_RULE", "targetPath": "/", "matchedUserAgents": [ "*" ], "matchedRule": null, "accessInference": "NOT_ESTABLISHED", "warnings": [], "sitemapDeclarations": { "urls": [ "https://www.usa.gov/sitemap.xml" ], "totalDeclared": 1, "listTruncated": false }, "outsideTokenInventory": { "tokens": [], "totalDistinct": 0, "listTruncated": false, "nonProductTokenCount": 0 }, "declaredUserAgentRuleCount": 0 }, "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "producedAt": "2026-10-23T12:23:17.749Z", "limitations": [ "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] } ], "limitations": [ "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] }, { "protocolExecutionId": "7032f20b-b9ec-44b7-a145-2a65ec3c7928", "protocolId": "policy.robots", "protocolVersion": "1.0.0", "stage": "POLICY", "required": false, "actor": { "actorId": "openai:gptbot", "provider": "OpenAI", "displayName": "GPTBot", "purpose": "TRAINING", "robotsPolicyApplicability": "APPLIES", "productToken": "GPTBot", "probeUserAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.0; +https://openai.com/gptbot)", "documentationUrl": "https://developers.openai.com/api/docs/bots", "documentationVerifiedAt": "2026-07-26" }, "startedAt": "2026-10-23T12:23:17.749Z", "finishedAt": "2026-10-23T12:23:17.749Z", "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "input": { "targetUrl": "https://www.usa.gov/", "robotsAcquisitionProtocolExecutionId": "031f4eac-2768-4b17-851e-6dd5c771f91c" }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "afd78db1-dda9-4f89-9eeb-ca31e14b3a93", "protocolExecutionId": "7032f20b-b9ec-44b7-a145-2a65ec3c7928", "kind": "ROBOTS_POLICY_DECISION", "result": "NOT_OBSERVED", "evidenceClass": "DERIVED", "value": { "actorId": "openai:gptbot", "actorPurpose": "TRAINING", "applicability": "APPLIES", "decision": "NO_MATCHING_RULE", "targetPath": "/", "matchedUserAgents": [ "*" ], "matchedRule": null, "accessInference": "NOT_ESTABLISHED", "warnings": [], "sitemapDeclarations": { "urls": [ "https://www.usa.gov/sitemap.xml" ], "totalDeclared": 1, "listTruncated": false }, "outsideTokenInventory": { "tokens": [], "totalDistinct": 0, "listTruncated": false, "nonProductTokenCount": 0 }, "declaredUserAgentRuleCount": 0 }, "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "producedAt": "2026-10-23T12:23:17.749Z", "limitations": [ "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] } ], "limitations": [ "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] }, { "protocolExecutionId": "1818f60b-7b1a-4bfc-8b0b-f1e8ed9964d3", "protocolId": "policy.robots", "protocolVersion": "1.0.0", "stage": "POLICY", "required": false, "actor": { "actorId": "anthropic:claudebot", "provider": "Anthropic", "displayName": "ClaudeBot", "purpose": "TRAINING", "robotsPolicyApplicability": "APPLIES", "productToken": "ClaudeBot", "probeUserAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)", "documentationUrl": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "documentationVerifiedAt": "2026-07-26" }, "startedAt": "2026-10-23T12:23:17.749Z", "finishedAt": "2026-10-23T12:23:17.749Z", "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "input": { "targetUrl": "https://www.usa.gov/", "robotsAcquisitionProtocolExecutionId": "031f4eac-2768-4b17-851e-6dd5c771f91c" }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "8fecde53-47eb-49a0-b169-984c4e467bae", "protocolExecutionId": "1818f60b-7b1a-4bfc-8b0b-f1e8ed9964d3", "kind": "ROBOTS_POLICY_DECISION", "result": "NOT_OBSERVED", "evidenceClass": "DERIVED", "value": { "actorId": "anthropic:claudebot", "actorPurpose": "TRAINING", "applicability": "APPLIES", "decision": "NO_MATCHING_RULE", "targetPath": "/", "matchedUserAgents": [ "*" ], "matchedRule": null, "accessInference": "NOT_ESTABLISHED", "warnings": [], "sitemapDeclarations": { "urls": [ "https://www.usa.gov/sitemap.xml" ], "totalDeclared": 1, "listTruncated": false }, "outsideTokenInventory": { "tokens": [], "totalDistinct": 0, "listTruncated": false, "nonProductTokenCount": 0 }, "declaredUserAgentRuleCount": 0 }, "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "producedAt": "2026-10-23T12:23:17.749Z", "limitations": [ "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] } ], "limitations": [ "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] }, { "protocolExecutionId": "6bbf2bc9-8d6f-4e6c-ba6b-4f5edaac5597", "protocolId": "policy.robots", "protocolVersion": "1.0.0", "stage": "POLICY", "required": false, "actor": { "actorId": "google:google-extended", "provider": "Google", "displayName": "Google-Extended", "purpose": "TRAINING_AND_GROUNDING", "robotsPolicyApplicability": "CONTROL_TOKEN", "productToken": "Google-Extended", "documentationUrl": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "documentationVerifiedAt": "2026-07-26" }, "startedAt": "2026-10-23T12:23:17.750Z", "finishedAt": "2026-10-23T12:23:17.750Z", "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "input": { "targetUrl": "https://www.usa.gov/", "robotsAcquisitionProtocolExecutionId": "031f4eac-2768-4b17-851e-6dd5c771f91c" }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "0e206c36-2b89-4e0c-b053-7a132922d384", "protocolExecutionId": "6bbf2bc9-8d6f-4e6c-ba6b-4f5edaac5597", "kind": "ROBOTS_POLICY_DECISION", "result": "NOT_OBSERVED", "evidenceClass": "DERIVED", "value": { "actorId": "google:google-extended", "actorPurpose": "TRAINING_AND_GROUNDING", "applicability": "CONTROL_TOKEN", "decision": "NO_MATCHING_RULE", "targetPath": "/", "matchedUserAgents": [ "*" ], "matchedRule": null, "accessInference": "NOT_ESTABLISHED", "warnings": [ "This product token expresses a downstream-use preference and is not a distinct HTTP crawler identity." ], "sitemapDeclarations": { "urls": [ "https://www.usa.gov/sitemap.xml" ], "totalDeclared": 1, "listTruncated": false }, "outsideTokenInventory": { "tokens": [], "totalDistinct": 0, "listTruncated": false, "nonProductTokenCount": 0 }, "declaredUserAgentRuleCount": 0 }, "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "producedAt": "2026-10-23T12:23:17.750Z", "limitations": [ "This product token expresses a downstream-use preference and is not a distinct HTTP crawler identity.", "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] } ], "limitations": [ "This product token expresses a downstream-use preference and is not a distinct HTTP crawler identity.", "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] }, { "protocolExecutionId": "4f576ddf-ccb7-441d-809a-85309defcfb9", "protocolId": "policy.robots", "protocolVersion": "1.0.0", "stage": "POLICY", "required": false, "actor": { "actorId": "openai:chatgpt-user", "provider": "OpenAI", "displayName": "ChatGPT-User", "purpose": "USER_ACTION", "robotsPolicyApplicability": "CONTEXT_DEPENDENT", "productToken": "ChatGPT-User", "probeUserAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot", "documentationUrl": "https://developers.openai.com/api/docs/bots", "documentationVerifiedAt": "2026-07-26" }, "startedAt": "2026-10-23T12:23:17.750Z", "finishedAt": "2026-10-23T12:23:17.750Z", "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "input": { "targetUrl": "https://www.usa.gov/", "robotsAcquisitionProtocolExecutionId": "031f4eac-2768-4b17-851e-6dd5c771f91c" }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "536f357d-d6ff-4987-a94d-ac720bf68d18", "protocolExecutionId": "4f576ddf-ccb7-441d-809a-85309defcfb9", "kind": "ROBOTS_POLICY_DECISION", "result": "NOT_OBSERVED", "evidenceClass": "DERIVED", "value": { "actorId": "openai:chatgpt-user", "actorPurpose": "USER_ACTION", "applicability": "CONTEXT_DEPENDENT", "decision": "NO_MATCHING_RULE", "targetPath": "/", "matchedUserAgents": [ "*" ], "matchedRule": null, "accessInference": "NOT_ESTABLISHED", "warnings": [ "This user-initiated actor may not use robots.txt as a universal access control; the directive is reported without inferring actual access." ], "sitemapDeclarations": { "urls": [ "https://www.usa.gov/sitemap.xml" ], "totalDeclared": 1, "listTruncated": false }, "outsideTokenInventory": { "tokens": [], "totalDistinct": 0, "listTruncated": false, "nonProductTokenCount": 0 }, "declaredUserAgentRuleCount": 0 }, "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "producedAt": "2026-10-23T12:23:17.750Z", "limitations": [ "This user-initiated actor may not use robots.txt as a universal access control; the directive is reported without inferring actual access.", "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] } ], "limitations": [ "This user-initiated actor may not use robots.txt as a universal access control; the directive is reported without inferring actual access.", "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] }, { "protocolExecutionId": "b5f96514-6b4f-4a76-a8ba-ead6aeb8ec98", "protocolId": "policy.robots", "protocolVersion": "1.0.0", "stage": "POLICY", "required": false, "actor": { "actorId": "anthropic:claude-user", "provider": "Anthropic", "displayName": "Claude-User", "purpose": "USER_ACTION", "robotsPolicyApplicability": "APPLIES", "productToken": "Claude-User", "probeUserAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Claude-User/1.0; +Claude-User@anthropic.com)", "documentationUrl": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "documentationVerifiedAt": "2026-07-26" }, "startedAt": "2026-10-23T12:23:17.750Z", "finishedAt": "2026-10-23T12:23:17.750Z", "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "input": { "targetUrl": "https://www.usa.gov/", "robotsAcquisitionProtocolExecutionId": "031f4eac-2768-4b17-851e-6dd5c771f91c" }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "beb6cd96-b7e6-4135-a0b8-67d4d5cef976", "protocolExecutionId": "b5f96514-6b4f-4a76-a8ba-ead6aeb8ec98", "kind": "ROBOTS_POLICY_DECISION", "result": "NOT_OBSERVED", "evidenceClass": "DERIVED", "value": { "actorId": "anthropic:claude-user", "actorPurpose": "USER_ACTION", "applicability": "APPLIES", "decision": "NO_MATCHING_RULE", "targetPath": "/", "matchedUserAgents": [ "*" ], "matchedRule": null, "accessInference": "NOT_ESTABLISHED", "warnings": [], "sitemapDeclarations": { "urls": [ "https://www.usa.gov/sitemap.xml" ], "totalDeclared": 1, "listTruncated": false }, "outsideTokenInventory": { "tokens": [], "totalDistinct": 0, "listTruncated": false, "nonProductTokenCount": 0 }, "declaredUserAgentRuleCount": 0 }, "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "producedAt": "2026-10-23T12:23:17.750Z", "limitations": [ "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] } ], "limitations": [ "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] }, { "protocolExecutionId": "0e5f38c9-50cd-43a9-8acc-dee6ab76d14b", "protocolId": "policy.robots", "protocolVersion": "1.0.0", "stage": "POLICY", "required": false, "actor": { "actorId": "perplexity:perplexity-user", "provider": "Perplexity", "displayName": "Perplexity-User", "purpose": "USER_ACTION", "robotsPolicyApplicability": "CONTEXT_DEPENDENT", "productToken": "Perplexity-User", "probeUserAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user)", "documentationUrl": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers", "documentationVerifiedAt": "2026-07-26" }, "startedAt": "2026-10-23T12:23:17.750Z", "finishedAt": "2026-10-23T12:23:17.750Z", "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "input": { "targetUrl": "https://www.usa.gov/", "robotsAcquisitionProtocolExecutionId": "031f4eac-2768-4b17-851e-6dd5c771f91c" }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "2c2b24fe-0d51-4849-bf27-107570aaf9ba", "protocolExecutionId": "0e5f38c9-50cd-43a9-8acc-dee6ab76d14b", "kind": "ROBOTS_POLICY_DECISION", "result": "NOT_OBSERVED", "evidenceClass": "DERIVED", "value": { "actorId": "perplexity:perplexity-user", "actorPurpose": "USER_ACTION", "applicability": "CONTEXT_DEPENDENT", "decision": "NO_MATCHING_RULE", "targetPath": "/", "matchedUserAgents": [ "*" ], "matchedRule": null, "accessInference": "NOT_ESTABLISHED", "warnings": [ "This user-initiated actor may not use robots.txt as a universal access control; the directive is reported without inferring actual access." ], "sitemapDeclarations": { "urls": [ "https://www.usa.gov/sitemap.xml" ], "totalDeclared": 1, "listTruncated": false }, "outsideTokenInventory": { "tokens": [], "totalDistinct": 0, "listTruncated": false, "nonProductTokenCount": 0 }, "declaredUserAgentRuleCount": 0 }, "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "producedAt": "2026-10-23T12:23:17.750Z", "limitations": [ "This user-initiated actor may not use robots.txt as a universal access control; the directive is reported without inferring actual access.", "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] } ], "limitations": [ "This user-initiated actor may not use robots.txt as a universal access control; the directive is reported without inferring actual access.", "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] }, { "protocolExecutionId": "ccd0c17e-ba26-4651-b1b3-a13305542763", "protocolId": "policy.robots", "protocolVersion": "1.0.0", "stage": "POLICY", "required": false, "actor": { "actorId": "google:google-agent", "provider": "Google", "displayName": "Google-Agent", "purpose": "USER_ACTION", "robotsPolicyApplicability": "CONTEXT_DEPENDENT", "productToken": "Google-Agent", "documentationUrl": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers", "documentationVerifiedAt": "2026-09-21" }, "startedAt": "2026-10-23T12:23:17.751Z", "finishedAt": "2026-10-23T12:23:17.751Z", "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "input": { "targetUrl": "https://www.usa.gov/", "robotsAcquisitionProtocolExecutionId": "031f4eac-2768-4b17-851e-6dd5c771f91c" }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "638e6099-d5bf-4b3b-8a3a-852331e7ff37", "protocolExecutionId": "ccd0c17e-ba26-4651-b1b3-a13305542763", "kind": "ROBOTS_POLICY_DECISION", "result": "NOT_OBSERVED", "evidenceClass": "DERIVED", "value": { "actorId": "google:google-agent", "actorPurpose": "USER_ACTION", "applicability": "CONTEXT_DEPENDENT", "decision": "NO_MATCHING_RULE", "targetPath": "/", "matchedUserAgents": [ "*" ], "matchedRule": null, "accessInference": "NOT_ESTABLISHED", "warnings": [ "This user-initiated actor may not use robots.txt as a universal access control; the directive is reported without inferring actual access." ], "sitemapDeclarations": { "urls": [ "https://www.usa.gov/sitemap.xml" ], "totalDeclared": 1, "listTruncated": false }, "outsideTokenInventory": { "tokens": [], "totalDistinct": 0, "listTruncated": false, "nonProductTokenCount": 0 }, "declaredUserAgentRuleCount": 0 }, "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "producedAt": "2026-10-23T12:23:17.751Z", "limitations": [ "This user-initiated actor may not use robots.txt as a universal access control; the directive is reported without inferring actual access.", "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] } ], "limitations": [ "This user-initiated actor may not use robots.txt as a universal access control; the directive is reported without inferring actual access.", "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] }, { "protocolExecutionId": "3abb3472-3a4a-42da-ab1a-09b40fa23120", "protocolId": "policy.robots", "protocolVersion": "1.0.0", "stage": "POLICY", "required": false, "actor": { "actorId": "openai:oai-adsbot", "provider": "OpenAI", "displayName": "OAI-AdsBot", "purpose": "ADS_VALIDATION", "robotsPolicyApplicability": "CONTEXT_DEPENDENT", "productToken": "OAI-AdsBot", "probeUserAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-AdsBot/1.0; +https://openai.com/adsbot", "documentationUrl": "https://developers.openai.com/api/docs/bots", "documentationVerifiedAt": "2026-09-21" }, "startedAt": "2026-10-23T12:23:17.751Z", "finishedAt": "2026-10-23T12:23:17.751Z", "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "input": { "targetUrl": "https://www.usa.gov/", "robotsAcquisitionProtocolExecutionId": "031f4eac-2768-4b17-851e-6dd5c771f91c" }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "21c19d27-f4d7-4472-8faf-93e529d73228", "protocolExecutionId": "3abb3472-3a4a-42da-ab1a-09b40fa23120", "kind": "ROBOTS_POLICY_DECISION", "result": "NOT_OBSERVED", "evidenceClass": "DERIVED", "value": { "actorId": "openai:oai-adsbot", "actorPurpose": "ADS_VALIDATION", "applicability": "CONTEXT_DEPENDENT", "decision": "NO_MATCHING_RULE", "targetPath": "/", "matchedUserAgents": [ "*" ], "matchedRule": null, "accessInference": "NOT_ESTABLISHED", "warnings": [ "No robots.txt behaviour is documented for this actor, so the directive is reported without inferring actual access." ], "sitemapDeclarations": { "urls": [ "https://www.usa.gov/sitemap.xml" ], "totalDeclared": 1, "listTruncated": false }, "outsideTokenInventory": { "tokens": [], "totalDistinct": 0, "listTruncated": false, "nonProductTokenCount": 0 }, "declaredUserAgentRuleCount": 0 }, "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "producedAt": "2026-10-23T12:23:17.751Z", "limitations": [ "No robots.txt behaviour is documented for this actor, so the directive is reported without inferring actual access.", "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] } ], "limitations": [ "No robots.txt behaviour is documented for this actor, so the directive is reported without inferring actual access.", "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] }, { "protocolExecutionId": "5ede9dcc-efcd-44a2-ae54-b02222a5e6c8", "protocolId": "policy.robots", "protocolVersion": "1.0.0", "stage": "POLICY", "required": false, "actor": { "actorId": "kagi:kagibot", "provider": "Kagi", "displayName": "Kagibot", "purpose": "SEARCH", "robotsPolicyApplicability": "APPLIES", "productToken": "Kagibot", "probeUserAgent": "Mozilla/5.0 (compatible; Kagibot/1.0; +https://kagi.com/bot)", "documentationUrl": "https://kagi.com/bot", "documentationVerifiedAt": "2026-10-14" }, "startedAt": "2026-10-23T12:23:17.751Z", "finishedAt": "2026-10-23T12:23:17.751Z", "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "input": { "targetUrl": "https://www.usa.gov/", "robotsAcquisitionProtocolExecutionId": "031f4eac-2768-4b17-851e-6dd5c771f91c" }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "ba18f42d-9983-4760-a33b-dce1b90a22ab", "protocolExecutionId": "5ede9dcc-efcd-44a2-ae54-b02222a5e6c8", "kind": "ROBOTS_POLICY_DECISION", "result": "NOT_OBSERVED", "evidenceClass": "DERIVED", "value": { "actorId": "kagi:kagibot", "actorPurpose": "SEARCH", "applicability": "APPLIES", "decision": "NO_MATCHING_RULE", "targetPath": "/", "matchedUserAgents": [ "*" ], "matchedRule": null, "accessInference": "NOT_ESTABLISHED", "warnings": [], "sitemapDeclarations": { "urls": [ "https://www.usa.gov/sitemap.xml" ], "totalDeclared": 1, "listTruncated": false }, "outsideTokenInventory": { "tokens": [], "totalDistinct": 0, "listTruncated": false, "nonProductTokenCount": 0 }, "declaredUserAgentRuleCount": 0 }, "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "producedAt": "2026-10-23T12:23:17.751Z", "limitations": [ "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] } ], "limitations": [ "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] }, { "protocolExecutionId": "16d4fbe6-5463-493a-90a8-6029bd56a88e", "protocolId": "policy.robots", "protocolVersion": "1.0.0", "stage": "POLICY", "required": false, "actor": { "actorId": "commoncrawl:ccbot", "provider": "Common Crawl", "displayName": "CCBot", "purpose": "UNSPECIFIED", "robotsPolicyApplicability": "APPLIES", "productToken": "CCBot", "probeUserAgent": "CCBot/2.0 (https://commoncrawl.org/faq/)", "documentationUrl": "https://commoncrawl.org/ccbot", "documentationVerifiedAt": "2026-10-06", "probed": false, "measuredInDefaultProfile": true }, "startedAt": "2026-10-23T12:23:17.751Z", "finishedAt": "2026-10-23T12:23:17.751Z", "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "input": { "targetUrl": "https://www.usa.gov/", "robotsAcquisitionProtocolExecutionId": "031f4eac-2768-4b17-851e-6dd5c771f91c" }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "573cc9b6-9d60-48b8-9bf1-279e5bf2d55b", "protocolExecutionId": "16d4fbe6-5463-493a-90a8-6029bd56a88e", "kind": "ROBOTS_POLICY_DECISION", "result": "NOT_OBSERVED", "evidenceClass": "DERIVED", "value": { "actorId": "commoncrawl:ccbot", "actorPurpose": "UNSPECIFIED", "applicability": "APPLIES", "decision": "NO_MATCHING_RULE", "targetPath": "/", "matchedUserAgents": [ "*" ], "matchedRule": null, "accessInference": "NOT_ESTABLISHED", "warnings": [], "sitemapDeclarations": { "urls": [ "https://www.usa.gov/sitemap.xml" ], "totalDeclared": 1, "listTruncated": false }, "outsideTokenInventory": { "tokens": [], "totalDistinct": 0, "listTruncated": false, "nonProductTokenCount": 0 }, "declaredUserAgentRuleCount": 0 }, "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "producedAt": "2026-10-23T12:23:17.751Z", "limitations": [ "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] } ], "limitations": [ "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] }, { "protocolExecutionId": "95b10d45-cc88-4b69-ac2f-724ab90424b3", "protocolId": "policy.robots", "protocolVersion": "1.0.0", "stage": "POLICY", "required": false, "actor": { "actorId": "microsoft:bingbot", "provider": "Microsoft", "displayName": "Bingbot", "purpose": "SEARCH", "robotsPolicyApplicability": "CONTEXT_DEPENDENT", "productToken": "bingbot", "documentationUrl": "https://www.bing.com/webmaster/help/which-crawlers-does-bing-use-8c184ec0", "documentationVerifiedAt": "2026-10-06", "probed": false, "measuredInDefaultProfile": true }, "startedAt": "2026-10-23T12:23:17.751Z", "finishedAt": "2026-10-23T12:23:17.751Z", "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "input": { "targetUrl": "https://www.usa.gov/", "robotsAcquisitionProtocolExecutionId": "031f4eac-2768-4b17-851e-6dd5c771f91c" }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "932c011a-6a5a-4296-8756-f77bc7a92bb3", "protocolExecutionId": "95b10d45-cc88-4b69-ac2f-724ab90424b3", "kind": "ROBOTS_POLICY_DECISION", "result": "NOT_OBSERVED", "evidenceClass": "DERIVED", "value": { "actorId": "microsoft:bingbot", "actorPurpose": "SEARCH", "applicability": "CONTEXT_DEPENDENT", "decision": "NO_MATCHING_RULE", "targetPath": "/", "matchedUserAgents": [ "*" ], "matchedRule": null, "accessInference": "NOT_ESTABLISHED", "warnings": [ "No robots.txt behaviour is documented for this actor, so the directive is reported without inferring actual access." ], "sitemapDeclarations": { "urls": [ "https://www.usa.gov/sitemap.xml" ], "totalDeclared": 1, "listTruncated": false }, "outsideTokenInventory": { "tokens": [], "totalDistinct": 0, "listTruncated": false, "nonProductTokenCount": 0 }, "declaredUserAgentRuleCount": 0 }, "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "producedAt": "2026-10-23T12:23:17.751Z", "limitations": [ "No robots.txt behaviour is documented for this actor, so the directive is reported without inferring actual access.", "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] } ], "limitations": [ "No robots.txt behaviour is documented for this actor, so the directive is reported without inferring actual access.", "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] }, { "protocolExecutionId": "b4e7dfbd-09ca-4386-ae02-2a245212cdf4", "protocolId": "policy.robots", "protocolVersion": "1.0.0", "stage": "POLICY", "required": false, "actor": { "actorId": "apple:applebot", "provider": "Apple", "displayName": "Applebot", "purpose": "SEARCH", "robotsPolicyApplicability": "APPLIES", "productToken": "Applebot", "probeUserAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15 (Applebot/0.1; +http://www.apple.com/go/applebot)", "documentationUrl": "https://support.apple.com/en-us/119829", "documentationVerifiedAt": "2026-10-06", "probed": false, "measuredInDefaultProfile": true }, "startedAt": "2026-10-23T12:23:17.752Z", "finishedAt": "2026-10-23T12:23:17.752Z", "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "input": { "targetUrl": "https://www.usa.gov/", "robotsAcquisitionProtocolExecutionId": "031f4eac-2768-4b17-851e-6dd5c771f91c" }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "6ba2ebc6-5e99-41df-907f-98a17e49ab35", "protocolExecutionId": "b4e7dfbd-09ca-4386-ae02-2a245212cdf4", "kind": "ROBOTS_POLICY_DECISION", "result": "NOT_OBSERVED", "evidenceClass": "DERIVED", "value": { "actorId": "apple:applebot", "actorPurpose": "SEARCH", "applicability": "APPLIES", "decision": "NO_MATCHING_RULE", "targetPath": "/", "matchedUserAgents": [ "*" ], "matchedRule": null, "accessInference": "NOT_ESTABLISHED", "warnings": [], "sitemapDeclarations": { "urls": [ "https://www.usa.gov/sitemap.xml" ], "totalDeclared": 1, "listTruncated": false }, "outsideTokenInventory": { "tokens": [], "totalDistinct": 0, "listTruncated": false, "nonProductTokenCount": 0 }, "declaredUserAgentRuleCount": 0 }, "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "producedAt": "2026-10-23T12:23:17.752Z", "limitations": [ "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] } ], "limitations": [ "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] }, { "protocolExecutionId": "60c2dd5d-b305-40ab-8373-9e13093dec78", "protocolId": "policy.robots", "protocolVersion": "1.0.0", "stage": "POLICY", "required": false, "actor": { "actorId": "apple:applebot-extended", "provider": "Apple", "displayName": "Applebot-Extended", "purpose": "TRAINING", "robotsPolicyApplicability": "CONTROL_TOKEN", "productToken": "Applebot-Extended", "documentationUrl": "https://support.apple.com/en-us/119829", "documentationVerifiedAt": "2026-10-06", "probed": false, "measuredInDefaultProfile": true }, "startedAt": "2026-10-23T12:23:17.752Z", "finishedAt": "2026-10-23T12:23:17.752Z", "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "input": { "targetUrl": "https://www.usa.gov/", "robotsAcquisitionProtocolExecutionId": "031f4eac-2768-4b17-851e-6dd5c771f91c" }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "4f6deb8a-c9dc-4913-99ae-3a9bf9d6015e", "protocolExecutionId": "60c2dd5d-b305-40ab-8373-9e13093dec78", "kind": "ROBOTS_POLICY_DECISION", "result": "NOT_OBSERVED", "evidenceClass": "DERIVED", "value": { "actorId": "apple:applebot-extended", "actorPurpose": "TRAINING", "applicability": "CONTROL_TOKEN", "decision": "NO_MATCHING_RULE", "targetPath": "/", "matchedUserAgents": [ "*" ], "matchedRule": null, "accessInference": "NOT_ESTABLISHED", "warnings": [ "This product token expresses a downstream-use preference and is not a distinct HTTP crawler identity." ], "sitemapDeclarations": { "urls": [ "https://www.usa.gov/sitemap.xml" ], "totalDeclared": 1, "listTruncated": false }, "outsideTokenInventory": { "tokens": [], "totalDistinct": 0, "listTruncated": false, "nonProductTokenCount": 0 }, "declaredUserAgentRuleCount": 0 }, "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "producedAt": "2026-10-23T12:23:17.752Z", "limitations": [ "This product token expresses a downstream-use preference and is not a distinct HTTP crawler identity.", "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] } ], "limitations": [ "This product token expresses a downstream-use preference and is not a distinct HTTP crawler identity.", "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] }, { "protocolExecutionId": "6c0784cc-c90a-49c0-b371-f927852afc46", "protocolId": "policy.robots", "protocolVersion": "1.0.0", "stage": "POLICY", "required": false, "actor": { "actorId": "duckduckgo:duckassistbot", "provider": "DuckDuckGo", "displayName": "DuckAssistBot", "purpose": "GROUNDING", "robotsPolicyApplicability": "APPLIES", "productToken": "DuckAssistBot", "probeUserAgent": "DuckAssistBot/1.2; (+http://duckduckgo.com/duckassistbot.html)", "documentationUrl": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot", "documentationVerifiedAt": "2026-10-06", "probed": false, "measuredInDefaultProfile": true }, "startedAt": "2026-10-23T12:23:17.752Z", "finishedAt": "2026-10-23T12:23:17.752Z", "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "input": { "targetUrl": "https://www.usa.gov/", "robotsAcquisitionProtocolExecutionId": "031f4eac-2768-4b17-851e-6dd5c771f91c" }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "ea9a6daa-1bf6-4925-8a44-aa149829066c", "protocolExecutionId": "6c0784cc-c90a-49c0-b371-f927852afc46", "kind": "ROBOTS_POLICY_DECISION", "result": "NOT_OBSERVED", "evidenceClass": "DERIVED", "value": { "actorId": "duckduckgo:duckassistbot", "actorPurpose": "GROUNDING", "applicability": "APPLIES", "decision": "NO_MATCHING_RULE", "targetPath": "/", "matchedUserAgents": [ "*" ], "matchedRule": null, "accessInference": "NOT_ESTABLISHED", "warnings": [], "sitemapDeclarations": { "urls": [ "https://www.usa.gov/sitemap.xml" ], "totalDeclared": 1, "listTruncated": false }, "outsideTokenInventory": { "tokens": [], "totalDistinct": 0, "listTruncated": false, "nonProductTokenCount": 0 }, "declaredUserAgentRuleCount": 0 }, "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "producedAt": "2026-10-23T12:23:17.752Z", "limitations": [ "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] } ], "limitations": [ "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] }, { "protocolExecutionId": "54adf426-1696-4db9-ad8a-73d3bc855210", "protocolId": "policy.robots", "protocolVersion": "1.0.0", "stage": "POLICY", "required": false, "actor": { "actorId": "diffbot:diffbot", "provider": "Diffbot", "displayName": "Diffbot", "purpose": "SEARCH", "robotsPolicyApplicability": "APPLIES", "productToken": "Diffbot", "documentationUrl": "https://www.diffbot.com/docs/crawl/faq/robots-txt", "documentationVerifiedAt": "2026-10-06", "probed": false, "measuredInDefaultProfile": true }, "startedAt": "2026-10-23T12:23:17.752Z", "finishedAt": "2026-10-23T12:23:17.752Z", "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "input": { "targetUrl": "https://www.usa.gov/", "robotsAcquisitionProtocolExecutionId": "031f4eac-2768-4b17-851e-6dd5c771f91c" }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "1fa61e47-3b98-493c-933c-128056683d32", "protocolExecutionId": "54adf426-1696-4db9-ad8a-73d3bc855210", "kind": "ROBOTS_POLICY_DECISION", "result": "NOT_OBSERVED", "evidenceClass": "DERIVED", "value": { "actorId": "diffbot:diffbot", "actorPurpose": "SEARCH", "applicability": "APPLIES", "decision": "NO_MATCHING_RULE", "targetPath": "/", "matchedUserAgents": [ "*" ], "matchedRule": null, "accessInference": "NOT_ESTABLISHED", "warnings": [], "sitemapDeclarations": { "urls": [ "https://www.usa.gov/sitemap.xml" ], "totalDeclared": 1, "listTruncated": false }, "outsideTokenInventory": { "tokens": [], "totalDistinct": 0, "listTruncated": false, "nonProductTokenCount": 0 }, "declaredUserAgentRuleCount": 0 }, "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "producedAt": "2026-10-23T12:23:17.752Z", "limitations": [ "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] } ], "limitations": [ "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] }, { "protocolExecutionId": "7f38ea6a-cc04-46de-9308-8796af354d4a", "protocolId": "policy.robots", "protocolVersion": "1.0.0", "stage": "POLICY", "required": false, "actor": { "actorId": "webzio:omgili", "provider": "Webz.io", "displayName": "omgili", "purpose": "UNSPECIFIED", "robotsPolicyApplicability": "APPLIES", "productToken": "omgili", "probeUserAgent": "omgili/0.5 +https://omgili.com", "aliasTokens": [ "omgilibot" ], "documentationUrl": "https://webz.io/blog/web-data/what-is-the-omgili-bot-and-why-is-it-crawling-your-website/", "documentationVerifiedAt": "2026-10-06", "probed": false, "measuredInDefaultProfile": true }, "startedAt": "2026-10-23T12:23:17.752Z", "finishedAt": "2026-10-23T12:23:17.752Z", "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "input": { "targetUrl": "https://www.usa.gov/", "robotsAcquisitionProtocolExecutionId": "031f4eac-2768-4b17-851e-6dd5c771f91c" }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "bd24efcb-753d-4ddd-9254-ea0904052ef3", "protocolExecutionId": "7f38ea6a-cc04-46de-9308-8796af354d4a", "kind": "ROBOTS_POLICY_DECISION", "result": "NOT_OBSERVED", "evidenceClass": "DERIVED", "value": { "actorId": "webzio:omgili", "actorPurpose": "UNSPECIFIED", "applicability": "APPLIES", "decision": "NO_MATCHING_RULE", "targetPath": "/", "matchedUserAgents": [ "*" ], "matchedRule": null, "accessInference": "NOT_ESTABLISHED", "warnings": [], "sitemapDeclarations": { "urls": [ "https://www.usa.gov/sitemap.xml" ], "totalDeclared": 1, "listTruncated": false }, "outsideTokenInventory": { "tokens": [], "totalDistinct": 0, "listTruncated": false, "nonProductTokenCount": 0 }, "declaredUserAgentRuleCount": 0 }, "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "producedAt": "2026-10-23T12:23:17.752Z", "limitations": [ "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] } ], "limitations": [ "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] }, { "protocolExecutionId": "8b922115-563b-4e3b-9b56-3ec059b930e0", "protocolId": "policy.robots", "protocolVersion": "1.0.0", "stage": "POLICY", "required": false, "actor": { "actorId": "allenai:ai2bot", "provider": "Allen Institute for AI", "displayName": "AI2Bot", "purpose": "TRAINING", "robotsPolicyApplicability": "CONTEXT_DEPENDENT", "productToken": "AI2Bot", "probeUserAgent": "Mozilla/5.0 (compatible) AI2Bot (+https://www.allenai.org/crawler)", "aliasTokens": [ "ai2bot-dolma" ], "documentationUrl": "https://allenai.org/crawler", "documentationVerifiedAt": "2026-10-06", "probed": false, "measuredInDefaultProfile": true }, "startedAt": "2026-10-23T12:23:17.753Z", "finishedAt": "2026-10-23T12:23:17.753Z", "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "input": { "targetUrl": "https://www.usa.gov/", "robotsAcquisitionProtocolExecutionId": "031f4eac-2768-4b17-851e-6dd5c771f91c" }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "0fcb0702-ccea-4a0f-9c46-48accb31a995", "protocolExecutionId": "8b922115-563b-4e3b-9b56-3ec059b930e0", "kind": "ROBOTS_POLICY_DECISION", "result": "NOT_OBSERVED", "evidenceClass": "DERIVED", "value": { "actorId": "allenai:ai2bot", "actorPurpose": "TRAINING", "applicability": "CONTEXT_DEPENDENT", "decision": "NO_MATCHING_RULE", "targetPath": "/", "matchedUserAgents": [ "*" ], "matchedRule": null, "accessInference": "NOT_ESTABLISHED", "warnings": [ "No robots.txt behaviour is documented for this actor, so the directive is reported without inferring actual access." ], "sitemapDeclarations": { "urls": [ "https://www.usa.gov/sitemap.xml" ], "totalDeclared": 1, "listTruncated": false }, "outsideTokenInventory": { "tokens": [], "totalDistinct": 0, "listTruncated": false, "nonProductTokenCount": 0 }, "declaredUserAgentRuleCount": 0 }, "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "producedAt": "2026-10-23T12:23:17.753Z", "limitations": [ "No robots.txt behaviour is documented for this actor, so the directive is reported without inferring actual access.", "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] } ], "limitations": [ "No robots.txt behaviour is documented for this actor, so the directive is reported without inferring actual access.", "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] }, { "protocolExecutionId": "e88688c8-e9bb-4ac5-bc08-81ba6b7c0e3a", "protocolId": "policy.robots", "protocolVersion": "1.0.0", "stage": "POLICY", "required": false, "actor": { "actorId": "hive:imagesiftbot", "provider": "Hive", "displayName": "ImagesiftBot", "purpose": "SEARCH", "robotsPolicyApplicability": "APPLIES", "probeUserAgent": "Mozilla/5.0 (compatible; ImagesiftBot; +imagesift.com)", "productToken": "ImagesiftBot", "documentationUrl": "https://imagesift.com/about", "documentationVerifiedAt": "2026-10-06", "probed": false, "measuredInDefaultProfile": true }, "startedAt": "2026-10-23T12:23:17.753Z", "finishedAt": "2026-10-23T12:23:17.753Z", "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "input": { "targetUrl": "https://www.usa.gov/", "robotsAcquisitionProtocolExecutionId": "031f4eac-2768-4b17-851e-6dd5c771f91c" }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "93dd07e8-4137-4a36-9ed2-588aa3713aac", "protocolExecutionId": "e88688c8-e9bb-4ac5-bc08-81ba6b7c0e3a", "kind": "ROBOTS_POLICY_DECISION", "result": "NOT_OBSERVED", "evidenceClass": "DERIVED", "value": { "actorId": "hive:imagesiftbot", "actorPurpose": "SEARCH", "applicability": "APPLIES", "decision": "NO_MATCHING_RULE", "targetPath": "/", "matchedUserAgents": [ "*" ], "matchedRule": null, "accessInference": "NOT_ESTABLISHED", "warnings": [], "sitemapDeclarations": { "urls": [ "https://www.usa.gov/sitemap.xml" ], "totalDeclared": 1, "listTruncated": false }, "outsideTokenInventory": { "tokens": [], "totalDistinct": 0, "listTruncated": false, "nonProductTokenCount": 0 }, "declaredUserAgentRuleCount": 0 }, "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "producedAt": "2026-10-23T12:23:17.753Z", "limitations": [ "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] } ], "limitations": [ "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] }, { "protocolExecutionId": "0d937fef-7df4-4208-bfc0-f7f89407f3a9", "protocolId": "policy.robots", "protocolVersion": "1.0.0", "stage": "POLICY", "required": false, "actor": { "actorId": "linkup:linkupbot", "provider": "Linkup", "displayName": "LinkupBot", "purpose": "SEARCH", "robotsPolicyApplicability": "APPLIES", "productToken": "LinkupBot", "probeUserAgent": "LinkupBot/1.0 (LinkupBot for web indexing; https://www.linkup.so/bot; bot@linkup.so)", "documentationUrl": "https://www.linkup.so/bot", "documentationVerifiedAt": "2026-10-06", "probed": false, "measuredInDefaultProfile": true }, "startedAt": "2026-10-23T12:23:17.753Z", "finishedAt": "2026-10-23T12:23:17.753Z", "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "input": { "targetUrl": "https://www.usa.gov/", "robotsAcquisitionProtocolExecutionId": "031f4eac-2768-4b17-851e-6dd5c771f91c" }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "2bcd8548-3f8e-4077-8866-b6f43d85f328", "protocolExecutionId": "0d937fef-7df4-4208-bfc0-f7f89407f3a9", "kind": "ROBOTS_POLICY_DECISION", "result": "NOT_OBSERVED", "evidenceClass": "DERIVED", "value": { "actorId": "linkup:linkupbot", "actorPurpose": "SEARCH", "applicability": "APPLIES", "decision": "NO_MATCHING_RULE", "targetPath": "/", "matchedUserAgents": [ "*" ], "matchedRule": null, "accessInference": "NOT_ESTABLISHED", "warnings": [], "sitemapDeclarations": { "urls": [ "https://www.usa.gov/sitemap.xml" ], "totalDeclared": 1, "listTruncated": false }, "outsideTokenInventory": { "tokens": [], "totalDistinct": 0, "listTruncated": false, "nonProductTokenCount": 0 }, "declaredUserAgentRuleCount": 0 }, "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "producedAt": "2026-10-23T12:23:17.753Z", "limitations": [ "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] } ], "limitations": [ "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] }, { "protocolExecutionId": "16670dce-2151-4eaf-b483-56233162c40e", "protocolId": "policy.robots", "protocolVersion": "1.0.0", "stage": "POLICY", "required": false, "actor": { "actorId": "google:google-cloudvertexbot", "provider": "Google", "displayName": "Google-CloudVertexBot", "purpose": "USER_ACTION", "robotsPolicyApplicability": "APPLIES", "productToken": "Google-CloudVertexBot", "documentationUrl": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "documentationVerifiedAt": "2026-10-06", "probed": false, "measuredInDefaultProfile": true }, "startedAt": "2026-10-23T12:23:17.753Z", "finishedAt": "2026-10-23T12:23:17.753Z", "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "input": { "targetUrl": "https://www.usa.gov/", "robotsAcquisitionProtocolExecutionId": "031f4eac-2768-4b17-851e-6dd5c771f91c" }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "0859d5e9-ab30-4c2d-b715-33118943e40e", "protocolExecutionId": "16670dce-2151-4eaf-b483-56233162c40e", "kind": "ROBOTS_POLICY_DECISION", "result": "NOT_OBSERVED", "evidenceClass": "DERIVED", "value": { "actorId": "google:google-cloudvertexbot", "actorPurpose": "USER_ACTION", "applicability": "APPLIES", "decision": "NO_MATCHING_RULE", "targetPath": "/", "matchedUserAgents": [ "*" ], "matchedRule": null, "accessInference": "NOT_ESTABLISHED", "warnings": [], "sitemapDeclarations": { "urls": [ "https://www.usa.gov/sitemap.xml" ], "totalDeclared": 1, "listTruncated": false }, "outsideTokenInventory": { "tokens": [], "totalDistinct": 0, "listTruncated": false, "nonProductTokenCount": 0 }, "declaredUserAgentRuleCount": 0 }, "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "producedAt": "2026-10-23T12:23:17.753Z", "limitations": [ "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] } ], "limitations": [ "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] }, { "protocolExecutionId": "7561f01a-5b2c-4e8b-ab62-c04fe7d3a31f", "protocolId": "policy.robots", "protocolVersion": "1.0.0", "stage": "POLICY", "required": false, "actor": { "actorId": "google:google-gemininotebook", "provider": "Google", "displayName": "Google-GeminiNotebook", "purpose": "USER_ACTION", "robotsPolicyApplicability": "CONTEXT_DEPENDENT", "productToken": "Google-GeminiNotebook", "documentationUrl": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers", "documentationVerifiedAt": "2026-10-06", "probed": false, "measuredInDefaultProfile": true }, "startedAt": "2026-10-23T12:23:17.753Z", "finishedAt": "2026-10-23T12:23:17.753Z", "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "input": { "targetUrl": "https://www.usa.gov/", "robotsAcquisitionProtocolExecutionId": "031f4eac-2768-4b17-851e-6dd5c771f91c" }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "60d0c2a4-77e8-4b0c-bee5-5834e24b6d0c", "protocolExecutionId": "7561f01a-5b2c-4e8b-ab62-c04fe7d3a31f", "kind": "ROBOTS_POLICY_DECISION", "result": "NOT_OBSERVED", "evidenceClass": "DERIVED", "value": { "actorId": "google:google-gemininotebook", "actorPurpose": "USER_ACTION", "applicability": "CONTEXT_DEPENDENT", "decision": "NO_MATCHING_RULE", "targetPath": "/", "matchedUserAgents": [ "*" ], "matchedRule": null, "accessInference": "NOT_ESTABLISHED", "warnings": [ "This user-initiated actor may not use robots.txt as a universal access control; the directive is reported without inferring actual access." ], "sitemapDeclarations": { "urls": [ "https://www.usa.gov/sitemap.xml" ], "totalDeclared": 1, "listTruncated": false }, "outsideTokenInventory": { "tokens": [], "totalDistinct": 0, "listTruncated": false, "nonProductTokenCount": 0 }, "declaredUserAgentRuleCount": 0 }, "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "producedAt": "2026-10-23T12:23:17.753Z", "limitations": [ "This user-initiated actor may not use robots.txt as a universal access control; the directive is reported without inferring actual access.", "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] } ], "limitations": [ "This user-initiated actor may not use robots.txt as a universal access control; the directive is reported without inferring actual access.", "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] }, { "protocolExecutionId": "0026771b-82bc-4369-9936-749a5fae813c", "protocolId": "policy.robots", "protocolVersion": "1.0.0", "stage": "POLICY", "required": false, "actor": { "actorId": "google:googleother", "provider": "Google", "displayName": "GoogleOther", "purpose": "UNSPECIFIED", "robotsPolicyApplicability": "APPLIES", "productToken": "GoogleOther", "documentationUrl": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "documentationVerifiedAt": "2026-10-06", "probed": false, "measuredInDefaultProfile": true }, "startedAt": "2026-10-23T12:23:17.753Z", "finishedAt": "2026-10-23T12:23:17.753Z", "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "input": { "targetUrl": "https://www.usa.gov/", "robotsAcquisitionProtocolExecutionId": "031f4eac-2768-4b17-851e-6dd5c771f91c" }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "baabc7fc-23bc-47c3-b203-f277e2b648ee", "protocolExecutionId": "0026771b-82bc-4369-9936-749a5fae813c", "kind": "ROBOTS_POLICY_DECISION", "result": "NOT_OBSERVED", "evidenceClass": "DERIVED", "value": { "actorId": "google:googleother", "actorPurpose": "UNSPECIFIED", "applicability": "APPLIES", "decision": "NO_MATCHING_RULE", "targetPath": "/", "matchedUserAgents": [ "*" ], "matchedRule": null, "accessInference": "NOT_ESTABLISHED", "warnings": [], "sitemapDeclarations": { "urls": [ "https://www.usa.gov/sitemap.xml" ], "totalDeclared": 1, "listTruncated": false }, "outsideTokenInventory": { "tokens": [], "totalDistinct": 0, "listTruncated": false, "nonProductTokenCount": 0 }, "declaredUserAgentRuleCount": 0 }, "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "producedAt": "2026-10-23T12:23:17.753Z", "limitations": [ "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] } ], "limitations": [ "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] }, { "protocolExecutionId": "2b11bb86-d1cb-4911-89ff-c041e2a450c2", "protocolId": "policy.robots", "protocolVersion": "1.0.0", "stage": "POLICY", "required": false, "actor": { "actorId": "firecrawl:firecrawlagent", "provider": "Firecrawl", "displayName": "FirecrawlAgent", "purpose": "UNSPECIFIED", "robotsPolicyApplicability": "APPLIES", "productToken": "FirecrawlAgent", "documentationUrl": "https://docs.firecrawl.dev/advanced-scraping-guide", "documentationVerifiedAt": "2026-10-06", "probed": false, "measuredInDefaultProfile": true }, "startedAt": "2026-10-23T12:23:17.754Z", "finishedAt": "2026-10-23T12:23:17.754Z", "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "input": { "targetUrl": "https://www.usa.gov/", "robotsAcquisitionProtocolExecutionId": "031f4eac-2768-4b17-851e-6dd5c771f91c" }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "6cd369aa-e20f-44ae-80e6-14de4d7e24ab", "protocolExecutionId": "2b11bb86-d1cb-4911-89ff-c041e2a450c2", "kind": "ROBOTS_POLICY_DECISION", "result": "NOT_OBSERVED", "evidenceClass": "DERIVED", "value": { "actorId": "firecrawl:firecrawlagent", "actorPurpose": "UNSPECIFIED", "applicability": "APPLIES", "decision": "NO_MATCHING_RULE", "targetPath": "/", "matchedUserAgents": [ "*" ], "matchedRule": null, "accessInference": "NOT_ESTABLISHED", "warnings": [], "sitemapDeclarations": { "urls": [ "https://www.usa.gov/sitemap.xml" ], "totalDeclared": 1, "listTruncated": false }, "outsideTokenInventory": { "tokens": [], "totalDistinct": 0, "listTruncated": false, "nonProductTokenCount": 0 }, "declaredUserAgentRuleCount": 0 }, "artifactRefs": [ "69a414b7-f69c-44f6-8c02-289242626038" ], "producedAt": "2026-10-23T12:23:17.754Z", "limitations": [ "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] } ], "limitations": [ "Sitemap: directives declared in robots.txt are recorded as declared; the listed URLs were not fetched and their presence was not validated." ] }, { "protocolExecutionId": "0e62a9ba-b10f-4582-83c5-d81813c84d1f", "protocolId": "policy.ai-usage-headers", "protocolVersion": "1.0.0", "stage": "POLICY", "required": false, "actor": { "actorId": "gm:ai-usage-header-probe", "provider": "Generative Metrics", "displayName": "Generative Metrics AI-usage header probe", "purpose": "UNSPECIFIED", "robotsPolicyApplicability": "NOT_APPLICABLE", "userAgent": "GenerativeMetrics/1.0 (+https://generativemetrics.com)" }, "startedAt": "2026-10-23T12:23:17.754Z", "finishedAt": "2026-10-23T12:23:17.754Z", "artifactRefs": [ "f0cb1e46-f740-49c5-8cbb-ab49da5046db" ], "input": { "targetUrl": "https://www.usa.gov/", "httpAcquisitionProtocolExecutionId": "cb943ad1-bd64-401f-9ebd-833a8f9b456b", "actorId": "gm:ai-usage-header-probe", "actorPurpose": "UNSPECIFIED", "policyScope": "OBSERVATION_ONLY" }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "f49adc84-4008-476b-b529-112c8d41359d", "protocolExecutionId": "0e62a9ba-b10f-4582-83c5-d81813c84d1f", "kind": "AI_USAGE_HEADER_POLICY", "result": "NOT_OBSERVED", "evidenceClass": "DIRECT", "value": { "policyScope": "OBSERVATION_ONLY", "valueShape": "PER_ACTOR_MAP", "perActor": { "openai:oai-searchbot": { "applicability": "APPLICABLE", "xRobotsTag": { "present": false, "tokens": [], "recognizedReviewTokens": [] }, "tdmReservation": { "present": false, "normalizedValue": null }, "recognizedSignalsPresent": false }, "anthropic:claude-searchbot": { "applicability": "APPLICABLE", "xRobotsTag": { "present": false, "tokens": [], "recognizedReviewTokens": [] }, "tdmReservation": { "present": false, "normalizedValue": null }, "recognizedSignalsPresent": false }, "google:googlebot": { "applicability": "APPLICABLE", "xRobotsTag": { "present": false, "tokens": [], "recognizedReviewTokens": [] }, "tdmReservation": { "present": false, "normalizedValue": null }, "recognizedSignalsPresent": false }, "perplexity:perplexitybot": { "applicability": "APPLICABLE", "xRobotsTag": { "present": false, "tokens": [], "recognizedReviewTokens": [] }, "tdmReservation": { "present": false, "normalizedValue": null }, "recognizedSignalsPresent": false }, "openai:gptbot": { "applicability": "APPLICABLE", "xRobotsTag": { "present": false, "tokens": [], "recognizedReviewTokens": [] }, "tdmReservation": { "present": false, "normalizedValue": null }, "recognizedSignalsPresent": false }, "anthropic:claudebot": { "applicability": "APPLICABLE", "xRobotsTag": { "present": false, "tokens": [], "recognizedReviewTokens": [] }, "tdmReservation": { "present": false, "normalizedValue": null }, "recognizedSignalsPresent": false }, "google:google-extended": { "applicability": "NOT_APPLICABLE", "xRobotsTag": { "present": false, "tokens": [], "recognizedReviewTokens": [] }, "tdmReservation": { "present": false, "normalizedValue": null }, "recognizedSignalsPresent": false }, "openai:chatgpt-user": { "applicability": "APPLICABLE", "xRobotsTag": { "present": false, "tokens": [], "recognizedReviewTokens": [] }, "tdmReservation": { "present": false, "normalizedValue": null }, "recognizedSignalsPresent": false }, "anthropic:claude-user": { "applicability": "APPLICABLE", "xRobotsTag": { "present": false, "tokens": [], "recognizedReviewTokens": [] }, "tdmReservation": { "present": false, "normalizedValue": null }, "recognizedSignalsPresent": false }, "perplexity:perplexity-user": { "applicability": "APPLICABLE", "xRobotsTag": { "present": false, "tokens": [], "recognizedReviewTokens": [] }, "tdmReservation": { "present": false, "normalizedValue": null }, "recognizedSignalsPresent": false }, "google:google-agent": { "applicability": "APPLICABLE", "xRobotsTag": { "present": false, "tokens": [], "recognizedReviewTokens": [] }, "tdmReservation": { "present": false, "normalizedValue": null }, "recognizedSignalsPresent": false }, "openai:oai-adsbot": { "applicability": "APPLICABLE", "xRobotsTag": { "present": false, "tokens": [], "recognizedReviewTokens": [] }, "tdmReservation": { "present": false, "normalizedValue": null }, "recognizedSignalsPresent": false }, "kagi:kagibot": { "applicability": "APPLICABLE", "xRobotsTag": { "present": false, "tokens": [], "recognizedReviewTokens": [] }, "tdmReservation": { "present": false, "normalizedValue": null }, "recognizedSignalsPresent": false }, "commoncrawl:ccbot": { "applicability": "APPLICABLE", "xRobotsTag": { "present": false, "tokens": [], "recognizedReviewTokens": [] }, "tdmReservation": { "present": false, "normalizedValue": null }, "recognizedSignalsPresent": false }, "microsoft:bingbot": { "applicability": "APPLICABLE", "xRobotsTag": { "present": false, "tokens": [], "recognizedReviewTokens": [] }, "tdmReservation": { "present": false, "normalizedValue": null }, "recognizedSignalsPresent": false }, "apple:applebot": { "applicability": "APPLICABLE", "xRobotsTag": { "present": false, "tokens": [], "recognizedReviewTokens": [] }, "tdmReservation": { "present": false, "normalizedValue": null }, "recognizedSignalsPresent": false }, "apple:applebot-extended": { "applicability": "NOT_APPLICABLE", "xRobotsTag": { "present": false, "tokens": [], "recognizedReviewTokens": [] }, "tdmReservation": { "present": false, "normalizedValue": null }, "recognizedSignalsPresent": false }, "duckduckgo:duckassistbot": { "applicability": "APPLICABLE", "xRobotsTag": { "present": false, "tokens": [], "recognizedReviewTokens": [] }, "tdmReservation": { "present": false, "normalizedValue": null }, "recognizedSignalsPresent": false }, "diffbot:diffbot": { "applicability": "APPLICABLE", "xRobotsTag": { "present": false, "tokens": [], "recognizedReviewTokens": [] }, "tdmReservation": { "present": false, "normalizedValue": null }, "recognizedSignalsPresent": false }, "webzio:omgili": { "applicability": "APPLICABLE", "xRobotsTag": { "present": false, "tokens": [], "recognizedReviewTokens": [] }, "tdmReservation": { "present": false, "normalizedValue": null }, "recognizedSignalsPresent": false }, "allenai:ai2bot": { "applicability": "APPLICABLE", "xRobotsTag": { "present": false, "tokens": [], "recognizedReviewTokens": [] }, "tdmReservation": { "present": false, "normalizedValue": null }, "recognizedSignalsPresent": false }, "hive:imagesiftbot": { "applicability": "APPLICABLE", "xRobotsTag": { "present": false, "tokens": [], "recognizedReviewTokens": [] }, "tdmReservation": { "present": false, "normalizedValue": null }, "recognizedSignalsPresent": false }, "linkup:linkupbot": { "applicability": "APPLICABLE", "xRobotsTag": { "present": false, "tokens": [], "recognizedReviewTokens": [] }, "tdmReservation": { "present": false, "normalizedValue": null }, "recognizedSignalsPresent": false }, "google:google-cloudvertexbot": { "applicability": "APPLICABLE", "xRobotsTag": { "present": false, "tokens": [], "recognizedReviewTokens": [] }, "tdmReservation": { "present": false, "normalizedValue": null }, "recognizedSignalsPresent": false }, "google:google-gemininotebook": { "applicability": "APPLICABLE", "xRobotsTag": { "present": false, "tokens": [], "recognizedReviewTokens": [] }, "tdmReservation": { "present": false, "normalizedValue": null }, "recognizedSignalsPresent": false }, "google:googleother": { "applicability": "APPLICABLE", "xRobotsTag": { "present": false, "tokens": [], "recognizedReviewTokens": [] }, "tdmReservation": { "present": false, "normalizedValue": null }, "recognizedSignalsPresent": false }, "firecrawl:firecrawlagent": { "applicability": "APPLICABLE", "xRobotsTag": { "present": false, "tokens": [], "recognizedReviewTokens": [] }, "tdmReservation": { "present": false, "normalizedValue": null }, "recognizedSignalsPresent": false } }, "recognizedSignalsPresent": false }, "artifactRefs": [ "f0cb1e46-f740-49c5-8cbb-ab49da5046db" ], "producedAt": "2026-10-23T12:23:17.754Z", "limitations": [ "Observed response headers are policy signals only; provider compliance, access denial, training behavior, indexing, and citation impact are not inferred." ] } ], "limitations": [ "Observed response headers are policy signals only; provider compliance, access denial, training behavior, indexing, and citation impact are not inferred." ] }, { "protocolExecutionId": "30bda123-20a0-4a48-b685-fdd74b3fc298", "protocolId": "access.actor-probe", "protocolVersion": "1.0.0", "stage": "ACQUISITION", "required": false, "actor": { "actorId": "openai:oai-searchbot", "provider": "OpenAI", "displayName": "OAI-SearchBot", "purpose": "SEARCH", "robotsPolicyApplicability": "APPLIES", "productToken": "OAI-SearchBot", "probeUserAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; OAI-SearchBot/1.0; +https://openai.com/searchbot)", "documentationUrl": "https://developers.openai.com/api/docs/bots", "documentationVerifiedAt": "2026-07-26" }, "startedAt": "2026-10-23T12:23:17.755Z", "finishedAt": "2026-10-23T12:23:17.755Z", "artifactRefs": [], "input": { "requestedUrl": "https://www.usa.gov/", "productToken": "OAI-SearchBot" }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "f9ea1163-2fa6-47c4-804a-da702e9e9d4d", "protocolExecutionId": "30bda123-20a0-4a48-b685-fdd74b3fc298", "kind": "ACTOR_ACCESS_RESULT", "result": "OBSERVED", "evidenceClass": "DIRECT", "value": { "actorId": "openai:oai-searchbot", "productToken": "OAI-SearchBot", "requestUserAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; OAI-SearchBot/1.0; +https://openai.com/searchbot)", "requestUserAgentSource": "VENDOR_PUBLISHED", "probeVantage": "GENERATIVE_METRICS_PROBE_REQUEST", "httpStatus": 200, "classification": "REACHED", "responseByteLength": 48647, "bodyTruncated": false, "failureCause": null, "finalUrl": null, "cfCacheStatus": null, "vercelCacheStatus": null, "confirmationProbe": null }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:00.409Z", "limitations": [ "Probe results are single-request HTTP observations taken at scan time from Generative Metrics’ own servers; they do not observe or predict what any provider’s systems actually experience or do.", "Classifications are categorical execution states derived from the HTTP status alone; they are never scores. Enforcement for required actors is also recorded as sealed access.actor-probe evidence, where an uncorroborated denial is recorded as indeterminate (WARNING) and a robots.txt policy denial remains the deterministic production blocker.", "Probes deliberately do not consult robots.txt (the audit measures the declared-versus-enforced gap) and send no disclosure header (measurement neutrality).", "Each probe carries the actor’s vendor-documented canonical User-Agent; where a vendor publishes no fixed User-Agent string, the documented product token is sent instead and the fallback is recorded on the probe row (userAgentSource).", "Bounded content facts (machine-extracted word count, one hash of the extracted text, bounded flags) are computed from the same probe response bodies and are compared with the scanner’s own fetch by the content.actor-extraction protocol. No response body text is retained, and these facts state what our probe received under an actor’s documented identity — never what that provider’s systems receive.", "Tracked AI-usage header values are parsed under a fixed, closed opt-out token grammar; only the parsed tokens, the agent-scoped map and an unparseable count are recorded — never the raw header string (values are truncated to 512 characters before parsing and anything off-grammar is only counted).", "Observed from Generative Metrics’ own network vantage with a single unauthenticated GET carrying the actor’s vendor-documented User-Agent; it does not prove what the provider’s systems experience or do." ] } ], "limitations": [ "Probe results are single-request HTTP observations taken at scan time from Generative Metrics’ own servers; they do not observe or predict what any provider’s systems actually experience or do.", "Classifications are categorical execution states derived from the HTTP status alone; they are never scores. Enforcement for required actors is also recorded as sealed access.actor-probe evidence, where an uncorroborated denial is recorded as indeterminate (WARNING) and a robots.txt policy denial remains the deterministic production blocker.", "Probes deliberately do not consult robots.txt (the audit measures the declared-versus-enforced gap) and send no disclosure header (measurement neutrality).", "Each probe carries the actor’s vendor-documented canonical User-Agent; where a vendor publishes no fixed User-Agent string, the documented product token is sent instead and the fallback is recorded on the probe row (userAgentSource).", "Bounded content facts (machine-extracted word count, one hash of the extracted text, bounded flags) are computed from the same probe response bodies and are compared with the scanner’s own fetch by the content.actor-extraction protocol. No response body text is retained, and these facts state what our probe received under an actor’s documented identity — never what that provider’s systems receive.", "Tracked AI-usage header values are parsed under a fixed, closed opt-out token grammar; only the parsed tokens, the agent-scoped map and an unparseable count are recorded — never the raw header string (values are truncated to 512 characters before parsing and anything off-grammar is only counted).", "Observed from Generative Metrics’ own network vantage with a single unauthenticated GET carrying the actor’s vendor-documented User-Agent; it does not prove what the provider’s systems experience or do." ] }, { "protocolExecutionId": "83b89a12-6b91-4d96-9e42-9158cdc96653", "protocolId": "access.actor-probe", "protocolVersion": "1.0.0", "stage": "ACQUISITION", "required": false, "actor": { "actorId": "anthropic:claude-searchbot", "provider": "Anthropic", "displayName": "Claude-SearchBot", "purpose": "SEARCH", "robotsPolicyApplicability": "APPLIES", "productToken": "Claude-SearchBot", "probeUserAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Claude-SearchBot/1.0; +https://www.anthropic.com/clausearchbot)", "documentationUrl": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "documentationVerifiedAt": "2026-07-26" }, "startedAt": "2026-10-23T12:23:17.755Z", "finishedAt": "2026-10-23T12:23:17.755Z", "artifactRefs": [], "input": { "requestedUrl": "https://www.usa.gov/", "productToken": "Claude-SearchBot" }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "031fe35a-244a-463e-b8dd-dd38e7342e5c", "protocolExecutionId": "83b89a12-6b91-4d96-9e42-9158cdc96653", "kind": "ACTOR_ACCESS_RESULT", "result": "OBSERVED", "evidenceClass": "DIRECT", "value": { "actorId": "anthropic:claude-searchbot", "productToken": "Claude-SearchBot", "requestUserAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Claude-SearchBot/1.0; +https://www.anthropic.com/clausearchbot)", "requestUserAgentSource": "VENDOR_PUBLISHED", "probeVantage": "GENERATIVE_METRICS_PROBE_REQUEST", "httpStatus": 200, "classification": "REACHED", "responseByteLength": 48647, "bodyTruncated": false, "failureCause": null, "finalUrl": null, "cfCacheStatus": null, "vercelCacheStatus": null, "confirmationProbe": null }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:01.340Z", "limitations": [ "Probe results are single-request HTTP observations taken at scan time from Generative Metrics’ own servers; they do not observe or predict what any provider’s systems actually experience or do.", "Classifications are categorical execution states derived from the HTTP status alone; they are never scores. Enforcement for required actors is also recorded as sealed access.actor-probe evidence, where an uncorroborated denial is recorded as indeterminate (WARNING) and a robots.txt policy denial remains the deterministic production blocker.", "Probes deliberately do not consult robots.txt (the audit measures the declared-versus-enforced gap) and send no disclosure header (measurement neutrality).", "Each probe carries the actor’s vendor-documented canonical User-Agent; where a vendor publishes no fixed User-Agent string, the documented product token is sent instead and the fallback is recorded on the probe row (userAgentSource).", "Bounded content facts (machine-extracted word count, one hash of the extracted text, bounded flags) are computed from the same probe response bodies and are compared with the scanner’s own fetch by the content.actor-extraction protocol. No response body text is retained, and these facts state what our probe received under an actor’s documented identity — never what that provider’s systems receive.", "Tracked AI-usage header values are parsed under a fixed, closed opt-out token grammar; only the parsed tokens, the agent-scoped map and an unparseable count are recorded — never the raw header string (values are truncated to 512 characters before parsing and anything off-grammar is only counted).", "Observed from Generative Metrics’ own network vantage with a single unauthenticated GET carrying the actor’s vendor-documented User-Agent; it does not prove what the provider’s systems experience or do." ] } ], "limitations": [ "Probe results are single-request HTTP observations taken at scan time from Generative Metrics’ own servers; they do not observe or predict what any provider’s systems actually experience or do.", "Classifications are categorical execution states derived from the HTTP status alone; they are never scores. Enforcement for required actors is also recorded as sealed access.actor-probe evidence, where an uncorroborated denial is recorded as indeterminate (WARNING) and a robots.txt policy denial remains the deterministic production blocker.", "Probes deliberately do not consult robots.txt (the audit measures the declared-versus-enforced gap) and send no disclosure header (measurement neutrality).", "Each probe carries the actor’s vendor-documented canonical User-Agent; where a vendor publishes no fixed User-Agent string, the documented product token is sent instead and the fallback is recorded on the probe row (userAgentSource).", "Bounded content facts (machine-extracted word count, one hash of the extracted text, bounded flags) are computed from the same probe response bodies and are compared with the scanner’s own fetch by the content.actor-extraction protocol. No response body text is retained, and these facts state what our probe received under an actor’s documented identity — never what that provider’s systems receive.", "Tracked AI-usage header values are parsed under a fixed, closed opt-out token grammar; only the parsed tokens, the agent-scoped map and an unparseable count are recorded — never the raw header string (values are truncated to 512 characters before parsing and anything off-grammar is only counted).", "Observed from Generative Metrics’ own network vantage with a single unauthenticated GET carrying the actor’s vendor-documented User-Agent; it does not prove what the provider’s systems experience or do." ] }, { "protocolExecutionId": "32e1e8fa-7051-4d2b-be7d-112caa3300c1", "protocolId": "access.actor-probe", "protocolVersion": "1.0.0", "stage": "ACQUISITION", "required": false, "actor": { "actorId": "google:googlebot", "provider": "Google", "displayName": "Googlebot", "purpose": "SEARCH", "robotsPolicyApplicability": "APPLIES", "productToken": "Googlebot", "probeUserAgent": "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", "documentationUrl": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "documentationVerifiedAt": "2026-07-26" }, "startedAt": "2026-10-23T12:23:17.755Z", "finishedAt": "2026-10-23T12:23:17.755Z", "artifactRefs": [], "input": { "requestedUrl": "https://www.usa.gov/", "productToken": "Googlebot" }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "16468b13-816c-42e0-b2a1-ff630989e5c8", "protocolExecutionId": "32e1e8fa-7051-4d2b-be7d-112caa3300c1", "kind": "ACTOR_ACCESS_RESULT", "result": "OBSERVED", "evidenceClass": "DIRECT", "value": { "actorId": "google:googlebot", "productToken": "Googlebot", "requestUserAgent": "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", "requestUserAgentSource": "VENDOR_PUBLISHED", "probeVantage": "GENERATIVE_METRICS_PROBE_REQUEST", "httpStatus": 200, "classification": "REACHED", "responseByteLength": 48647, "bodyTruncated": false, "failureCause": null, "finalUrl": null, "cfCacheStatus": null, "vercelCacheStatus": null, "confirmationProbe": null }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:02.386Z", "limitations": [ "Probe results are single-request HTTP observations taken at scan time from Generative Metrics’ own servers; they do not observe or predict what any provider’s systems actually experience or do.", "Classifications are categorical execution states derived from the HTTP status alone; they are never scores. Enforcement for required actors is also recorded as sealed access.actor-probe evidence, where an uncorroborated denial is recorded as indeterminate (WARNING) and a robots.txt policy denial remains the deterministic production blocker.", "Probes deliberately do not consult robots.txt (the audit measures the declared-versus-enforced gap) and send no disclosure header (measurement neutrality).", "Each probe carries the actor’s vendor-documented canonical User-Agent; where a vendor publishes no fixed User-Agent string, the documented product token is sent instead and the fallback is recorded on the probe row (userAgentSource).", "Bounded content facts (machine-extracted word count, one hash of the extracted text, bounded flags) are computed from the same probe response bodies and are compared with the scanner’s own fetch by the content.actor-extraction protocol. No response body text is retained, and these facts state what our probe received under an actor’s documented identity — never what that provider’s systems receive.", "Tracked AI-usage header values are parsed under a fixed, closed opt-out token grammar; only the parsed tokens, the agent-scoped map and an unparseable count are recorded — never the raw header string (values are truncated to 512 characters before parsing and anything off-grammar is only counted).", "Observed from Generative Metrics’ own network vantage with a single unauthenticated GET carrying the actor’s vendor-documented User-Agent; it does not prove what the provider’s systems experience or do." ] } ], "limitations": [ "Probe results are single-request HTTP observations taken at scan time from Generative Metrics’ own servers; they do not observe or predict what any provider’s systems actually experience or do.", "Classifications are categorical execution states derived from the HTTP status alone; they are never scores. Enforcement for required actors is also recorded as sealed access.actor-probe evidence, where an uncorroborated denial is recorded as indeterminate (WARNING) and a robots.txt policy denial remains the deterministic production blocker.", "Probes deliberately do not consult robots.txt (the audit measures the declared-versus-enforced gap) and send no disclosure header (measurement neutrality).", "Each probe carries the actor’s vendor-documented canonical User-Agent; where a vendor publishes no fixed User-Agent string, the documented product token is sent instead and the fallback is recorded on the probe row (userAgentSource).", "Bounded content facts (machine-extracted word count, one hash of the extracted text, bounded flags) are computed from the same probe response bodies and are compared with the scanner’s own fetch by the content.actor-extraction protocol. No response body text is retained, and these facts state what our probe received under an actor’s documented identity — never what that provider’s systems receive.", "Tracked AI-usage header values are parsed under a fixed, closed opt-out token grammar; only the parsed tokens, the agent-scoped map and an unparseable count are recorded — never the raw header string (values are truncated to 512 characters before parsing and anything off-grammar is only counted).", "Observed from Generative Metrics’ own network vantage with a single unauthenticated GET carrying the actor’s vendor-documented User-Agent; it does not prove what the provider’s systems experience or do." ] }, { "protocolExecutionId": "673135c1-7a83-4475-8d05-4abf2d40bd2f", "protocolId": "access.actor-probe", "protocolVersion": "1.0.0", "stage": "ACQUISITION", "required": false, "actor": { "actorId": "perplexity:perplexitybot", "provider": "Perplexity", "displayName": "PerplexityBot", "purpose": "SEARCH", "robotsPolicyApplicability": "APPLIES", "productToken": "PerplexityBot", "probeUserAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)", "documentationUrl": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers", "documentationVerifiedAt": "2026-07-26" }, "startedAt": "2026-10-23T12:23:17.755Z", "finishedAt": "2026-10-23T12:23:17.755Z", "artifactRefs": [], "input": { "requestedUrl": "https://www.usa.gov/", "productToken": "PerplexityBot" }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "a0ade623-111d-4603-b34d-006ee963cc4e", "protocolExecutionId": "673135c1-7a83-4475-8d05-4abf2d40bd2f", "kind": "ACTOR_ACCESS_RESULT", "result": "OBSERVED", "evidenceClass": "DIRECT", "value": { "actorId": "perplexity:perplexitybot", "productToken": "PerplexityBot", "requestUserAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)", "requestUserAgentSource": "VENDOR_PUBLISHED", "probeVantage": "GENERATIVE_METRICS_PROBE_REQUEST", "httpStatus": 200, "classification": "REACHED", "responseByteLength": 48647, "bodyTruncated": false, "failureCause": null, "finalUrl": null, "cfCacheStatus": null, "vercelCacheStatus": null, "confirmationProbe": null }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:03.352Z", "limitations": [ "Probe results are single-request HTTP observations taken at scan time from Generative Metrics’ own servers; they do not observe or predict what any provider’s systems actually experience or do.", "Classifications are categorical execution states derived from the HTTP status alone; they are never scores. Enforcement for required actors is also recorded as sealed access.actor-probe evidence, where an uncorroborated denial is recorded as indeterminate (WARNING) and a robots.txt policy denial remains the deterministic production blocker.", "Probes deliberately do not consult robots.txt (the audit measures the declared-versus-enforced gap) and send no disclosure header (measurement neutrality).", "Each probe carries the actor’s vendor-documented canonical User-Agent; where a vendor publishes no fixed User-Agent string, the documented product token is sent instead and the fallback is recorded on the probe row (userAgentSource).", "Bounded content facts (machine-extracted word count, one hash of the extracted text, bounded flags) are computed from the same probe response bodies and are compared with the scanner’s own fetch by the content.actor-extraction protocol. No response body text is retained, and these facts state what our probe received under an actor’s documented identity — never what that provider’s systems receive.", "Tracked AI-usage header values are parsed under a fixed, closed opt-out token grammar; only the parsed tokens, the agent-scoped map and an unparseable count are recorded — never the raw header string (values are truncated to 512 characters before parsing and anything off-grammar is only counted).", "Observed from Generative Metrics’ own network vantage with a single unauthenticated GET carrying the actor’s vendor-documented User-Agent; it does not prove what the provider’s systems experience or do." ] } ], "limitations": [ "Probe results are single-request HTTP observations taken at scan time from Generative Metrics’ own servers; they do not observe or predict what any provider’s systems actually experience or do.", "Classifications are categorical execution states derived from the HTTP status alone; they are never scores. Enforcement for required actors is also recorded as sealed access.actor-probe evidence, where an uncorroborated denial is recorded as indeterminate (WARNING) and a robots.txt policy denial remains the deterministic production blocker.", "Probes deliberately do not consult robots.txt (the audit measures the declared-versus-enforced gap) and send no disclosure header (measurement neutrality).", "Each probe carries the actor’s vendor-documented canonical User-Agent; where a vendor publishes no fixed User-Agent string, the documented product token is sent instead and the fallback is recorded on the probe row (userAgentSource).", "Bounded content facts (machine-extracted word count, one hash of the extracted text, bounded flags) are computed from the same probe response bodies and are compared with the scanner’s own fetch by the content.actor-extraction protocol. No response body text is retained, and these facts state what our probe received under an actor’s documented identity — never what that provider’s systems receive.", "Tracked AI-usage header values are parsed under a fixed, closed opt-out token grammar; only the parsed tokens, the agent-scoped map and an unparseable count are recorded — never the raw header string (values are truncated to 512 characters before parsing and anything off-grammar is only counted).", "Observed from Generative Metrics’ own network vantage with a single unauthenticated GET carrying the actor’s vendor-documented User-Agent; it does not prove what the provider’s systems experience or do." ] }, { "protocolExecutionId": "55d9cc00-5e99-42c1-bc64-d810c519ab45", "protocolId": "access.actor-probe", "protocolVersion": "1.0.0", "stage": "ACQUISITION", "required": false, "actor": { "actorId": "openai:gptbot", "provider": "OpenAI", "displayName": "GPTBot", "purpose": "TRAINING", "robotsPolicyApplicability": "APPLIES", "productToken": "GPTBot", "probeUserAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.0; +https://openai.com/gptbot)", "documentationUrl": "https://developers.openai.com/api/docs/bots", "documentationVerifiedAt": "2026-07-26" }, "startedAt": "2026-10-23T12:23:17.755Z", "finishedAt": "2026-10-23T12:23:17.755Z", "artifactRefs": [], "input": { "requestedUrl": "https://www.usa.gov/", "productToken": "GPTBot" }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "1e6598cf-3eea-427a-bb0b-7b9d4369167f", "protocolExecutionId": "55d9cc00-5e99-42c1-bc64-d810c519ab45", "kind": "ACTOR_ACCESS_RESULT", "result": "OBSERVED", "evidenceClass": "DIRECT", "value": { "actorId": "openai:gptbot", "productToken": "GPTBot", "requestUserAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.0; +https://openai.com/gptbot)", "requestUserAgentSource": "VENDOR_PUBLISHED", "probeVantage": "GENERATIVE_METRICS_PROBE_REQUEST", "httpStatus": 200, "classification": "REACHED", "responseByteLength": 48647, "bodyTruncated": false, "failureCause": null, "finalUrl": null, "cfCacheStatus": null, "vercelCacheStatus": null, "confirmationProbe": null }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:04.420Z", "limitations": [ "Probe results are single-request HTTP observations taken at scan time from Generative Metrics’ own servers; they do not observe or predict what any provider’s systems actually experience or do.", "Classifications are categorical execution states derived from the HTTP status alone; they are never scores. Enforcement for required actors is also recorded as sealed access.actor-probe evidence, where an uncorroborated denial is recorded as indeterminate (WARNING) and a robots.txt policy denial remains the deterministic production blocker.", "Probes deliberately do not consult robots.txt (the audit measures the declared-versus-enforced gap) and send no disclosure header (measurement neutrality).", "Each probe carries the actor’s vendor-documented canonical User-Agent; where a vendor publishes no fixed User-Agent string, the documented product token is sent instead and the fallback is recorded on the probe row (userAgentSource).", "Bounded content facts (machine-extracted word count, one hash of the extracted text, bounded flags) are computed from the same probe response bodies and are compared with the scanner’s own fetch by the content.actor-extraction protocol. No response body text is retained, and these facts state what our probe received under an actor’s documented identity — never what that provider’s systems receive.", "Tracked AI-usage header values are parsed under a fixed, closed opt-out token grammar; only the parsed tokens, the agent-scoped map and an unparseable count are recorded — never the raw header string (values are truncated to 512 characters before parsing and anything off-grammar is only counted).", "Observed from Generative Metrics’ own network vantage with a single unauthenticated GET carrying the actor’s vendor-documented User-Agent; it does not prove what the provider’s systems experience or do." ] } ], "limitations": [ "Probe results are single-request HTTP observations taken at scan time from Generative Metrics’ own servers; they do not observe or predict what any provider’s systems actually experience or do.", "Classifications are categorical execution states derived from the HTTP status alone; they are never scores. Enforcement for required actors is also recorded as sealed access.actor-probe evidence, where an uncorroborated denial is recorded as indeterminate (WARNING) and a robots.txt policy denial remains the deterministic production blocker.", "Probes deliberately do not consult robots.txt (the audit measures the declared-versus-enforced gap) and send no disclosure header (measurement neutrality).", "Each probe carries the actor’s vendor-documented canonical User-Agent; where a vendor publishes no fixed User-Agent string, the documented product token is sent instead and the fallback is recorded on the probe row (userAgentSource).", "Bounded content facts (machine-extracted word count, one hash of the extracted text, bounded flags) are computed from the same probe response bodies and are compared with the scanner’s own fetch by the content.actor-extraction protocol. No response body text is retained, and these facts state what our probe received under an actor’s documented identity — never what that provider’s systems receive.", "Tracked AI-usage header values are parsed under a fixed, closed opt-out token grammar; only the parsed tokens, the agent-scoped map and an unparseable count are recorded — never the raw header string (values are truncated to 512 characters before parsing and anything off-grammar is only counted).", "Observed from Generative Metrics’ own network vantage with a single unauthenticated GET carrying the actor’s vendor-documented User-Agent; it does not prove what the provider’s systems experience or do." ] }, { "protocolExecutionId": "8873f9b2-a782-48b0-b4af-dd79516a8fd8", "protocolId": "access.actor-probe", "protocolVersion": "1.0.0", "stage": "ACQUISITION", "required": false, "actor": { "actorId": "anthropic:claudebot", "provider": "Anthropic", "displayName": "ClaudeBot", "purpose": "TRAINING", "robotsPolicyApplicability": "APPLIES", "productToken": "ClaudeBot", "probeUserAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)", "documentationUrl": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "documentationVerifiedAt": "2026-07-26" }, "startedAt": "2026-10-23T12:23:17.755Z", "finishedAt": "2026-10-23T12:23:17.755Z", "artifactRefs": [], "input": { "requestedUrl": "https://www.usa.gov/", "productToken": "ClaudeBot" }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "615393d7-5e3d-42f8-a988-cbf139a913bf", "protocolExecutionId": "8873f9b2-a782-48b0-b4af-dd79516a8fd8", "kind": "ACTOR_ACCESS_RESULT", "result": "OBSERVED", "evidenceClass": "DIRECT", "value": { "actorId": "anthropic:claudebot", "productToken": "ClaudeBot", "requestUserAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)", "requestUserAgentSource": "VENDOR_PUBLISHED", "probeVantage": "GENERATIVE_METRICS_PROBE_REQUEST", "httpStatus": 200, "classification": "REACHED", "responseByteLength": 48647, "bodyTruncated": false, "failureCause": null, "finalUrl": null, "cfCacheStatus": null, "vercelCacheStatus": null, "confirmationProbe": null }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:05.358Z", "limitations": [ "Probe results are single-request HTTP observations taken at scan time from Generative Metrics’ own servers; they do not observe or predict what any provider’s systems actually experience or do.", "Classifications are categorical execution states derived from the HTTP status alone; they are never scores. Enforcement for required actors is also recorded as sealed access.actor-probe evidence, where an uncorroborated denial is recorded as indeterminate (WARNING) and a robots.txt policy denial remains the deterministic production blocker.", "Probes deliberately do not consult robots.txt (the audit measures the declared-versus-enforced gap) and send no disclosure header (measurement neutrality).", "Each probe carries the actor’s vendor-documented canonical User-Agent; where a vendor publishes no fixed User-Agent string, the documented product token is sent instead and the fallback is recorded on the probe row (userAgentSource).", "Bounded content facts (machine-extracted word count, one hash of the extracted text, bounded flags) are computed from the same probe response bodies and are compared with the scanner’s own fetch by the content.actor-extraction protocol. No response body text is retained, and these facts state what our probe received under an actor’s documented identity — never what that provider’s systems receive.", "Tracked AI-usage header values are parsed under a fixed, closed opt-out token grammar; only the parsed tokens, the agent-scoped map and an unparseable count are recorded — never the raw header string (values are truncated to 512 characters before parsing and anything off-grammar is only counted).", "Observed from Generative Metrics’ own network vantage with a single unauthenticated GET carrying the actor’s vendor-documented User-Agent; it does not prove what the provider’s systems experience or do." ] } ], "limitations": [ "Probe results are single-request HTTP observations taken at scan time from Generative Metrics’ own servers; they do not observe or predict what any provider’s systems actually experience or do.", "Classifications are categorical execution states derived from the HTTP status alone; they are never scores. Enforcement for required actors is also recorded as sealed access.actor-probe evidence, where an uncorroborated denial is recorded as indeterminate (WARNING) and a robots.txt policy denial remains the deterministic production blocker.", "Probes deliberately do not consult robots.txt (the audit measures the declared-versus-enforced gap) and send no disclosure header (measurement neutrality).", "Each probe carries the actor’s vendor-documented canonical User-Agent; where a vendor publishes no fixed User-Agent string, the documented product token is sent instead and the fallback is recorded on the probe row (userAgentSource).", "Bounded content facts (machine-extracted word count, one hash of the extracted text, bounded flags) are computed from the same probe response bodies and are compared with the scanner’s own fetch by the content.actor-extraction protocol. No response body text is retained, and these facts state what our probe received under an actor’s documented identity — never what that provider’s systems receive.", "Tracked AI-usage header values are parsed under a fixed, closed opt-out token grammar; only the parsed tokens, the agent-scoped map and an unparseable count are recorded — never the raw header string (values are truncated to 512 characters before parsing and anything off-grammar is only counted).", "Observed from Generative Metrics’ own network vantage with a single unauthenticated GET carrying the actor’s vendor-documented User-Agent; it does not prove what the provider’s systems experience or do." ] }, { "protocolExecutionId": "9c005061-3800-4c84-a1cf-b7353f946bc4", "protocolId": "access.actor-probe", "protocolVersion": "1.0.0", "stage": "ACQUISITION", "required": false, "actor": { "actorId": "google:google-extended", "provider": "Google", "displayName": "Google-Extended", "purpose": "TRAINING_AND_GROUNDING", "robotsPolicyApplicability": "CONTROL_TOKEN", "productToken": "Google-Extended", "documentationUrl": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers", "documentationVerifiedAt": "2026-07-26" }, "startedAt": "2026-10-23T12:23:17.755Z", "finishedAt": "2026-10-23T12:23:17.755Z", "artifactRefs": [], "input": { "requestedUrl": "https://www.usa.gov/", "productToken": "Google-Extended" }, "failure": null, "state": "NOT_APPLICABLE", "observations": [], "limitations": [ "Probe results are single-request HTTP observations taken at scan time from Generative Metrics’ own servers; they do not observe or predict what any provider’s systems actually experience or do.", "Classifications are categorical execution states derived from the HTTP status alone; they are never scores. Enforcement for required actors is also recorded as sealed access.actor-probe evidence, where an uncorroborated denial is recorded as indeterminate (WARNING) and a robots.txt policy denial remains the deterministic production blocker.", "Probes deliberately do not consult robots.txt (the audit measures the declared-versus-enforced gap) and send no disclosure header (measurement neutrality).", "Each probe carries the actor’s vendor-documented canonical User-Agent; where a vendor publishes no fixed User-Agent string, the documented product token is sent instead and the fallback is recorded on the probe row (userAgentSource).", "Bounded content facts (machine-extracted word count, one hash of the extracted text, bounded flags) are computed from the same probe response bodies and are compared with the scanner’s own fetch by the content.actor-extraction protocol. No response body text is retained, and these facts state what our probe received under an actor’s documented identity — never what that provider’s systems receive.", "Tracked AI-usage header values are parsed under a fixed, closed opt-out token grammar; only the parsed tokens, the agent-scoped map and an unparseable count are recorded — never the raw header string (values are truncated to 512 characters before parsing and anything off-grammar is only counted).", "CONTROL_TOKEN: convention — this token is a declared preference, not a distinct HTTP crawler identity, so no request was sent." ] }, { "protocolExecutionId": "6914ff36-98c5-410c-a2a6-ec8d46d289c7", "protocolId": "access.actor-probe", "protocolVersion": "1.0.0", "stage": "ACQUISITION", "required": false, "actor": { "actorId": "openai:chatgpt-user", "provider": "OpenAI", "displayName": "ChatGPT-User", "purpose": "USER_ACTION", "robotsPolicyApplicability": "CONTEXT_DEPENDENT", "productToken": "ChatGPT-User", "probeUserAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot", "documentationUrl": "https://developers.openai.com/api/docs/bots", "documentationVerifiedAt": "2026-07-26" }, "startedAt": "2026-10-23T12:23:17.755Z", "finishedAt": "2026-10-23T12:23:17.755Z", "artifactRefs": [], "input": { "requestedUrl": "https://www.usa.gov/", "productToken": "ChatGPT-User" }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "9d12455e-8528-437d-ae3c-f705c925857f", "protocolExecutionId": "6914ff36-98c5-410c-a2a6-ec8d46d289c7", "kind": "ACTOR_ACCESS_RESULT", "result": "OBSERVED", "evidenceClass": "DIRECT", "value": { "actorId": "openai:chatgpt-user", "productToken": "ChatGPT-User", "requestUserAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot", "requestUserAgentSource": "VENDOR_PUBLISHED", "probeVantage": "GENERATIVE_METRICS_PROBE_REQUEST", "httpStatus": 200, "classification": "REACHED", "responseByteLength": 48647, "bodyTruncated": false, "failureCause": null, "finalUrl": null, "cfCacheStatus": null, "vercelCacheStatus": null, "confirmationProbe": null }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:06.351Z", "limitations": [ "Probe results are single-request HTTP observations taken at scan time from Generative Metrics’ own servers; they do not observe or predict what any provider’s systems actually experience or do.", "Classifications are categorical execution states derived from the HTTP status alone; they are never scores. Enforcement for required actors is also recorded as sealed access.actor-probe evidence, where an uncorroborated denial is recorded as indeterminate (WARNING) and a robots.txt policy denial remains the deterministic production blocker.", "Probes deliberately do not consult robots.txt (the audit measures the declared-versus-enforced gap) and send no disclosure header (measurement neutrality).", "Each probe carries the actor’s vendor-documented canonical User-Agent; where a vendor publishes no fixed User-Agent string, the documented product token is sent instead and the fallback is recorded on the probe row (userAgentSource).", "Bounded content facts (machine-extracted word count, one hash of the extracted text, bounded flags) are computed from the same probe response bodies and are compared with the scanner’s own fetch by the content.actor-extraction protocol. No response body text is retained, and these facts state what our probe received under an actor’s documented identity — never what that provider’s systems receive.", "Tracked AI-usage header values are parsed under a fixed, closed opt-out token grammar; only the parsed tokens, the agent-scoped map and an unparseable count are recorded — never the raw header string (values are truncated to 512 characters before parsing and anything off-grammar is only counted).", "Observed from Generative Metrics’ own network vantage with a single unauthenticated GET carrying the actor’s vendor-documented User-Agent; it does not prove what the provider’s systems experience or do." ] } ], "limitations": [ "Probe results are single-request HTTP observations taken at scan time from Generative Metrics’ own servers; they do not observe or predict what any provider’s systems actually experience or do.", "Classifications are categorical execution states derived from the HTTP status alone; they are never scores. Enforcement for required actors is also recorded as sealed access.actor-probe evidence, where an uncorroborated denial is recorded as indeterminate (WARNING) and a robots.txt policy denial remains the deterministic production blocker.", "Probes deliberately do not consult robots.txt (the audit measures the declared-versus-enforced gap) and send no disclosure header (measurement neutrality).", "Each probe carries the actor’s vendor-documented canonical User-Agent; where a vendor publishes no fixed User-Agent string, the documented product token is sent instead and the fallback is recorded on the probe row (userAgentSource).", "Bounded content facts (machine-extracted word count, one hash of the extracted text, bounded flags) are computed from the same probe response bodies and are compared with the scanner’s own fetch by the content.actor-extraction protocol. No response body text is retained, and these facts state what our probe received under an actor’s documented identity — never what that provider’s systems receive.", "Tracked AI-usage header values are parsed under a fixed, closed opt-out token grammar; only the parsed tokens, the agent-scoped map and an unparseable count are recorded — never the raw header string (values are truncated to 512 characters before parsing and anything off-grammar is only counted).", "Observed from Generative Metrics’ own network vantage with a single unauthenticated GET carrying the actor’s vendor-documented User-Agent; it does not prove what the provider’s systems experience or do." ] }, { "protocolExecutionId": "c55dce3e-f851-4445-a579-4286a3ef82d8", "protocolId": "access.actor-probe", "protocolVersion": "1.0.0", "stage": "ACQUISITION", "required": false, "actor": { "actorId": "anthropic:claude-user", "provider": "Anthropic", "displayName": "Claude-User", "purpose": "USER_ACTION", "robotsPolicyApplicability": "APPLIES", "productToken": "Claude-User", "probeUserAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Claude-User/1.0; +Claude-User@anthropic.com)", "documentationUrl": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler", "documentationVerifiedAt": "2026-07-26" }, "startedAt": "2026-10-23T12:23:17.755Z", "finishedAt": "2026-10-23T12:23:17.755Z", "artifactRefs": [], "input": { "requestedUrl": "https://www.usa.gov/", "productToken": "Claude-User" }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "742188f4-8837-4738-ab93-e667e57940ee", "protocolExecutionId": "c55dce3e-f851-4445-a579-4286a3ef82d8", "kind": "ACTOR_ACCESS_RESULT", "result": "OBSERVED", "evidenceClass": "DIRECT", "value": { "actorId": "anthropic:claude-user", "productToken": "Claude-User", "requestUserAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Claude-User/1.0; +Claude-User@anthropic.com)", "requestUserAgentSource": "VENDOR_PUBLISHED", "probeVantage": "GENERATIVE_METRICS_PROBE_REQUEST", "httpStatus": 200, "classification": "REACHED", "responseByteLength": 48647, "bodyTruncated": false, "failureCause": null, "finalUrl": null, "cfCacheStatus": null, "vercelCacheStatus": null, "confirmationProbe": null }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:07.426Z", "limitations": [ "Probe results are single-request HTTP observations taken at scan time from Generative Metrics’ own servers; they do not observe or predict what any provider’s systems actually experience or do.", "Classifications are categorical execution states derived from the HTTP status alone; they are never scores. Enforcement for required actors is also recorded as sealed access.actor-probe evidence, where an uncorroborated denial is recorded as indeterminate (WARNING) and a robots.txt policy denial remains the deterministic production blocker.", "Probes deliberately do not consult robots.txt (the audit measures the declared-versus-enforced gap) and send no disclosure header (measurement neutrality).", "Each probe carries the actor’s vendor-documented canonical User-Agent; where a vendor publishes no fixed User-Agent string, the documented product token is sent instead and the fallback is recorded on the probe row (userAgentSource).", "Bounded content facts (machine-extracted word count, one hash of the extracted text, bounded flags) are computed from the same probe response bodies and are compared with the scanner’s own fetch by the content.actor-extraction protocol. No response body text is retained, and these facts state what our probe received under an actor’s documented identity — never what that provider’s systems receive.", "Tracked AI-usage header values are parsed under a fixed, closed opt-out token grammar; only the parsed tokens, the agent-scoped map and an unparseable count are recorded — never the raw header string (values are truncated to 512 characters before parsing and anything off-grammar is only counted).", "Observed from Generative Metrics’ own network vantage with a single unauthenticated GET carrying the actor’s vendor-documented User-Agent; it does not prove what the provider’s systems experience or do." ] } ], "limitations": [ "Probe results are single-request HTTP observations taken at scan time from Generative Metrics’ own servers; they do not observe or predict what any provider’s systems actually experience or do.", "Classifications are categorical execution states derived from the HTTP status alone; they are never scores. Enforcement for required actors is also recorded as sealed access.actor-probe evidence, where an uncorroborated denial is recorded as indeterminate (WARNING) and a robots.txt policy denial remains the deterministic production blocker.", "Probes deliberately do not consult robots.txt (the audit measures the declared-versus-enforced gap) and send no disclosure header (measurement neutrality).", "Each probe carries the actor’s vendor-documented canonical User-Agent; where a vendor publishes no fixed User-Agent string, the documented product token is sent instead and the fallback is recorded on the probe row (userAgentSource).", "Bounded content facts (machine-extracted word count, one hash of the extracted text, bounded flags) are computed from the same probe response bodies and are compared with the scanner’s own fetch by the content.actor-extraction protocol. No response body text is retained, and these facts state what our probe received under an actor’s documented identity — never what that provider’s systems receive.", "Tracked AI-usage header values are parsed under a fixed, closed opt-out token grammar; only the parsed tokens, the agent-scoped map and an unparseable count are recorded — never the raw header string (values are truncated to 512 characters before parsing and anything off-grammar is only counted).", "Observed from Generative Metrics’ own network vantage with a single unauthenticated GET carrying the actor’s vendor-documented User-Agent; it does not prove what the provider’s systems experience or do." ] }, { "protocolExecutionId": "6d047dce-74a8-4741-97fc-0c19e703e2b1", "protocolId": "access.actor-probe", "protocolVersion": "1.0.0", "stage": "ACQUISITION", "required": false, "actor": { "actorId": "perplexity:perplexity-user", "provider": "Perplexity", "displayName": "Perplexity-User", "purpose": "USER_ACTION", "robotsPolicyApplicability": "CONTEXT_DEPENDENT", "productToken": "Perplexity-User", "probeUserAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user)", "documentationUrl": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers", "documentationVerifiedAt": "2026-07-26" }, "startedAt": "2026-10-23T12:23:17.755Z", "finishedAt": "2026-10-23T12:23:17.755Z", "artifactRefs": [], "input": { "requestedUrl": "https://www.usa.gov/", "productToken": "Perplexity-User" }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "5502debe-5b50-4147-9ca3-d465ea78622c", "protocolExecutionId": "6d047dce-74a8-4741-97fc-0c19e703e2b1", "kind": "ACTOR_ACCESS_RESULT", "result": "OBSERVED", "evidenceClass": "DIRECT", "value": { "actorId": "perplexity:perplexity-user", "productToken": "Perplexity-User", "requestUserAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user)", "requestUserAgentSource": "VENDOR_PUBLISHED", "probeVantage": "GENERATIVE_METRICS_PROBE_REQUEST", "httpStatus": 200, "classification": "REACHED", "responseByteLength": 48647, "bodyTruncated": false, "failureCause": null, "finalUrl": null, "cfCacheStatus": null, "vercelCacheStatus": null, "confirmationProbe": null }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:08.382Z", "limitations": [ "Probe results are single-request HTTP observations taken at scan time from Generative Metrics’ own servers; they do not observe or predict what any provider’s systems actually experience or do.", "Classifications are categorical execution states derived from the HTTP status alone; they are never scores. Enforcement for required actors is also recorded as sealed access.actor-probe evidence, where an uncorroborated denial is recorded as indeterminate (WARNING) and a robots.txt policy denial remains the deterministic production blocker.", "Probes deliberately do not consult robots.txt (the audit measures the declared-versus-enforced gap) and send no disclosure header (measurement neutrality).", "Each probe carries the actor’s vendor-documented canonical User-Agent; where a vendor publishes no fixed User-Agent string, the documented product token is sent instead and the fallback is recorded on the probe row (userAgentSource).", "Bounded content facts (machine-extracted word count, one hash of the extracted text, bounded flags) are computed from the same probe response bodies and are compared with the scanner’s own fetch by the content.actor-extraction protocol. No response body text is retained, and these facts state what our probe received under an actor’s documented identity — never what that provider’s systems receive.", "Tracked AI-usage header values are parsed under a fixed, closed opt-out token grammar; only the parsed tokens, the agent-scoped map and an unparseable count are recorded — never the raw header string (values are truncated to 512 characters before parsing and anything off-grammar is only counted).", "Observed from Generative Metrics’ own network vantage with a single unauthenticated GET carrying the actor’s vendor-documented User-Agent; it does not prove what the provider’s systems experience or do." ] } ], "limitations": [ "Probe results are single-request HTTP observations taken at scan time from Generative Metrics’ own servers; they do not observe or predict what any provider’s systems actually experience or do.", "Classifications are categorical execution states derived from the HTTP status alone; they are never scores. Enforcement for required actors is also recorded as sealed access.actor-probe evidence, where an uncorroborated denial is recorded as indeterminate (WARNING) and a robots.txt policy denial remains the deterministic production blocker.", "Probes deliberately do not consult robots.txt (the audit measures the declared-versus-enforced gap) and send no disclosure header (measurement neutrality).", "Each probe carries the actor’s vendor-documented canonical User-Agent; where a vendor publishes no fixed User-Agent string, the documented product token is sent instead and the fallback is recorded on the probe row (userAgentSource).", "Bounded content facts (machine-extracted word count, one hash of the extracted text, bounded flags) are computed from the same probe response bodies and are compared with the scanner’s own fetch by the content.actor-extraction protocol. No response body text is retained, and these facts state what our probe received under an actor’s documented identity — never what that provider’s systems receive.", "Tracked AI-usage header values are parsed under a fixed, closed opt-out token grammar; only the parsed tokens, the agent-scoped map and an unparseable count are recorded — never the raw header string (values are truncated to 512 characters before parsing and anything off-grammar is only counted).", "Observed from Generative Metrics’ own network vantage with a single unauthenticated GET carrying the actor’s vendor-documented User-Agent; it does not prove what the provider’s systems experience or do." ] }, { "protocolExecutionId": "721ea368-4176-4e11-867f-04df2d1a58e8", "protocolId": "access.actor-probe", "protocolVersion": "1.0.0", "stage": "ACQUISITION", "required": false, "actor": { "actorId": "google:google-agent", "provider": "Google", "displayName": "Google-Agent", "purpose": "USER_ACTION", "robotsPolicyApplicability": "CONTEXT_DEPENDENT", "productToken": "Google-Agent", "documentationUrl": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers", "documentationVerifiedAt": "2026-09-21" }, "startedAt": "2026-10-23T12:23:17.755Z", "finishedAt": "2026-10-23T12:23:17.755Z", "artifactRefs": [], "input": { "requestedUrl": "https://www.usa.gov/", "productToken": "Google-Agent" }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "b6cbe837-c56e-4b2a-98e1-a8e5721c3b94", "protocolExecutionId": "721ea368-4176-4e11-867f-04df2d1a58e8", "kind": "ACTOR_ACCESS_RESULT", "result": "OBSERVED", "evidenceClass": "DIRECT", "value": { "actorId": "google:google-agent", "productToken": "Google-Agent", "requestUserAgent": "Google-Agent", "requestUserAgentSource": "PRODUCT_TOKEN_FALLBACK", "probeVantage": "GENERATIVE_METRICS_PROBE_REQUEST", "httpStatus": 200, "classification": "REACHED", "responseByteLength": 48647, "bodyTruncated": false, "failureCause": null, "finalUrl": null, "cfCacheStatus": null, "vercelCacheStatus": null, "confirmationProbe": null }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:09.367Z", "limitations": [ "Probe results are single-request HTTP observations taken at scan time from Generative Metrics’ own servers; they do not observe or predict what any provider’s systems actually experience or do.", "Classifications are categorical execution states derived from the HTTP status alone; they are never scores. Enforcement for required actors is also recorded as sealed access.actor-probe evidence, where an uncorroborated denial is recorded as indeterminate (WARNING) and a robots.txt policy denial remains the deterministic production blocker.", "Probes deliberately do not consult robots.txt (the audit measures the declared-versus-enforced gap) and send no disclosure header (measurement neutrality).", "Each probe carries the actor’s vendor-documented canonical User-Agent; where a vendor publishes no fixed User-Agent string, the documented product token is sent instead and the fallback is recorded on the probe row (userAgentSource).", "Bounded content facts (machine-extracted word count, one hash of the extracted text, bounded flags) are computed from the same probe response bodies and are compared with the scanner’s own fetch by the content.actor-extraction protocol. No response body text is retained, and these facts state what our probe received under an actor’s documented identity — never what that provider’s systems receive.", "Tracked AI-usage header values are parsed under a fixed, closed opt-out token grammar; only the parsed tokens, the agent-scoped map and an unparseable count are recorded — never the raw header string (values are truncated to 512 characters before parsing and anything off-grammar is only counted).", "Observed from Generative Metrics’ own network vantage with a single unauthenticated GET carrying the actor’s vendor-documented User-Agent; it does not prove what the provider’s systems experience or do." ] } ], "limitations": [ "Probe results are single-request HTTP observations taken at scan time from Generative Metrics’ own servers; they do not observe or predict what any provider’s systems actually experience or do.", "Classifications are categorical execution states derived from the HTTP status alone; they are never scores. Enforcement for required actors is also recorded as sealed access.actor-probe evidence, where an uncorroborated denial is recorded as indeterminate (WARNING) and a robots.txt policy denial remains the deterministic production blocker.", "Probes deliberately do not consult robots.txt (the audit measures the declared-versus-enforced gap) and send no disclosure header (measurement neutrality).", "Each probe carries the actor’s vendor-documented canonical User-Agent; where a vendor publishes no fixed User-Agent string, the documented product token is sent instead and the fallback is recorded on the probe row (userAgentSource).", "Bounded content facts (machine-extracted word count, one hash of the extracted text, bounded flags) are computed from the same probe response bodies and are compared with the scanner’s own fetch by the content.actor-extraction protocol. No response body text is retained, and these facts state what our probe received under an actor’s documented identity — never what that provider’s systems receive.", "Tracked AI-usage header values are parsed under a fixed, closed opt-out token grammar; only the parsed tokens, the agent-scoped map and an unparseable count are recorded — never the raw header string (values are truncated to 512 characters before parsing and anything off-grammar is only counted).", "Observed from Generative Metrics’ own network vantage with a single unauthenticated GET carrying the actor’s vendor-documented User-Agent; it does not prove what the provider’s systems experience or do." ] }, { "protocolExecutionId": "d40956f3-ba70-466b-b569-9c8b7e9256c9", "protocolId": "access.actor-probe", "protocolVersion": "1.0.0", "stage": "ACQUISITION", "required": false, "actor": { "actorId": "openai:oai-adsbot", "provider": "OpenAI", "displayName": "OAI-AdsBot", "purpose": "ADS_VALIDATION", "robotsPolicyApplicability": "CONTEXT_DEPENDENT", "productToken": "OAI-AdsBot", "probeUserAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-AdsBot/1.0; +https://openai.com/adsbot", "documentationUrl": "https://developers.openai.com/api/docs/bots", "documentationVerifiedAt": "2026-09-21" }, "startedAt": "2026-10-23T12:23:17.755Z", "finishedAt": "2026-10-23T12:23:17.755Z", "artifactRefs": [], "input": { "requestedUrl": "https://www.usa.gov/", "productToken": "OAI-AdsBot" }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "3ac389da-a72b-4d7a-a12c-491ee639f335", "protocolExecutionId": "d40956f3-ba70-466b-b569-9c8b7e9256c9", "kind": "ACTOR_ACCESS_RESULT", "result": "OBSERVED", "evidenceClass": "DIRECT", "value": { "actorId": "openai:oai-adsbot", "productToken": "OAI-AdsBot", "requestUserAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-AdsBot/1.0; +https://openai.com/adsbot", "requestUserAgentSource": "VENDOR_PUBLISHED", "probeVantage": "GENERATIVE_METRICS_PROBE_REQUEST", "httpStatus": 200, "classification": "REACHED", "responseByteLength": 48647, "bodyTruncated": false, "failureCause": null, "finalUrl": null, "cfCacheStatus": null, "vercelCacheStatus": null, "confirmationProbe": null }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:10.439Z", "limitations": [ "Probe results are single-request HTTP observations taken at scan time from Generative Metrics’ own servers; they do not observe or predict what any provider’s systems actually experience or do.", "Classifications are categorical execution states derived from the HTTP status alone; they are never scores. Enforcement for required actors is also recorded as sealed access.actor-probe evidence, where an uncorroborated denial is recorded as indeterminate (WARNING) and a robots.txt policy denial remains the deterministic production blocker.", "Probes deliberately do not consult robots.txt (the audit measures the declared-versus-enforced gap) and send no disclosure header (measurement neutrality).", "Each probe carries the actor’s vendor-documented canonical User-Agent; where a vendor publishes no fixed User-Agent string, the documented product token is sent instead and the fallback is recorded on the probe row (userAgentSource).", "Bounded content facts (machine-extracted word count, one hash of the extracted text, bounded flags) are computed from the same probe response bodies and are compared with the scanner’s own fetch by the content.actor-extraction protocol. No response body text is retained, and these facts state what our probe received under an actor’s documented identity — never what that provider’s systems receive.", "Tracked AI-usage header values are parsed under a fixed, closed opt-out token grammar; only the parsed tokens, the agent-scoped map and an unparseable count are recorded — never the raw header string (values are truncated to 512 characters before parsing and anything off-grammar is only counted).", "Observed from Generative Metrics’ own network vantage with a single unauthenticated GET carrying the actor’s vendor-documented User-Agent; it does not prove what the provider’s systems experience or do." ] } ], "limitations": [ "Probe results are single-request HTTP observations taken at scan time from Generative Metrics’ own servers; they do not observe or predict what any provider’s systems actually experience or do.", "Classifications are categorical execution states derived from the HTTP status alone; they are never scores. Enforcement for required actors is also recorded as sealed access.actor-probe evidence, where an uncorroborated denial is recorded as indeterminate (WARNING) and a robots.txt policy denial remains the deterministic production blocker.", "Probes deliberately do not consult robots.txt (the audit measures the declared-versus-enforced gap) and send no disclosure header (measurement neutrality).", "Each probe carries the actor’s vendor-documented canonical User-Agent; where a vendor publishes no fixed User-Agent string, the documented product token is sent instead and the fallback is recorded on the probe row (userAgentSource).", "Bounded content facts (machine-extracted word count, one hash of the extracted text, bounded flags) are computed from the same probe response bodies and are compared with the scanner’s own fetch by the content.actor-extraction protocol. No response body text is retained, and these facts state what our probe received under an actor’s documented identity — never what that provider’s systems receive.", "Tracked AI-usage header values are parsed under a fixed, closed opt-out token grammar; only the parsed tokens, the agent-scoped map and an unparseable count are recorded — never the raw header string (values are truncated to 512 characters before parsing and anything off-grammar is only counted).", "Observed from Generative Metrics’ own network vantage with a single unauthenticated GET carrying the actor’s vendor-documented User-Agent; it does not prove what the provider’s systems experience or do." ] }, { "protocolExecutionId": "2929abb2-8d7d-417c-9a6c-11143a28324f", "protocolId": "access.actor-probe", "protocolVersion": "1.0.0", "stage": "ACQUISITION", "required": false, "actor": { "actorId": "kagi:kagibot", "provider": "Kagi", "displayName": "Kagibot", "purpose": "SEARCH", "robotsPolicyApplicability": "APPLIES", "productToken": "Kagibot", "probeUserAgent": "Mozilla/5.0 (compatible; Kagibot/1.0; +https://kagi.com/bot)", "documentationUrl": "https://kagi.com/bot", "documentationVerifiedAt": "2026-10-14" }, "startedAt": "2026-10-23T12:23:17.755Z", "finishedAt": "2026-10-23T12:23:17.755Z", "artifactRefs": [], "input": { "requestedUrl": "https://www.usa.gov/", "productToken": "Kagibot" }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "f952433c-2bf8-4877-bb98-2ec2a547e894", "protocolExecutionId": "2929abb2-8d7d-417c-9a6c-11143a28324f", "kind": "ACTOR_ACCESS_RESULT", "result": "OBSERVED", "evidenceClass": "DIRECT", "value": { "actorId": "kagi:kagibot", "productToken": "Kagibot", "requestUserAgent": "Mozilla/5.0 (compatible; Kagibot/1.0; +https://kagi.com/bot)", "requestUserAgentSource": "VENDOR_PUBLISHED", "probeVantage": "GENERATIVE_METRICS_PROBE_REQUEST", "httpStatus": 200, "classification": "REACHED", "responseByteLength": 48647, "bodyTruncated": false, "failureCause": null, "finalUrl": null, "cfCacheStatus": null, "vercelCacheStatus": null, "confirmationProbe": null }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:11.418Z", "limitations": [ "Probe results are single-request HTTP observations taken at scan time from Generative Metrics’ own servers; they do not observe or predict what any provider’s systems actually experience or do.", "Classifications are categorical execution states derived from the HTTP status alone; they are never scores. Enforcement for required actors is also recorded as sealed access.actor-probe evidence, where an uncorroborated denial is recorded as indeterminate (WARNING) and a robots.txt policy denial remains the deterministic production blocker.", "Probes deliberately do not consult robots.txt (the audit measures the declared-versus-enforced gap) and send no disclosure header (measurement neutrality).", "Each probe carries the actor’s vendor-documented canonical User-Agent; where a vendor publishes no fixed User-Agent string, the documented product token is sent instead and the fallback is recorded on the probe row (userAgentSource).", "Bounded content facts (machine-extracted word count, one hash of the extracted text, bounded flags) are computed from the same probe response bodies and are compared with the scanner’s own fetch by the content.actor-extraction protocol. No response body text is retained, and these facts state what our probe received under an actor’s documented identity — never what that provider’s systems receive.", "Tracked AI-usage header values are parsed under a fixed, closed opt-out token grammar; only the parsed tokens, the agent-scoped map and an unparseable count are recorded — never the raw header string (values are truncated to 512 characters before parsing and anything off-grammar is only counted).", "Observed from Generative Metrics’ own network vantage with a single unauthenticated GET carrying the actor’s vendor-documented User-Agent; it does not prove what the provider’s systems experience or do." ] } ], "limitations": [ "Probe results are single-request HTTP observations taken at scan time from Generative Metrics’ own servers; they do not observe or predict what any provider’s systems actually experience or do.", "Classifications are categorical execution states derived from the HTTP status alone; they are never scores. Enforcement for required actors is also recorded as sealed access.actor-probe evidence, where an uncorroborated denial is recorded as indeterminate (WARNING) and a robots.txt policy denial remains the deterministic production blocker.", "Probes deliberately do not consult robots.txt (the audit measures the declared-versus-enforced gap) and send no disclosure header (measurement neutrality).", "Each probe carries the actor’s vendor-documented canonical User-Agent; where a vendor publishes no fixed User-Agent string, the documented product token is sent instead and the fallback is recorded on the probe row (userAgentSource).", "Bounded content facts (machine-extracted word count, one hash of the extracted text, bounded flags) are computed from the same probe response bodies and are compared with the scanner’s own fetch by the content.actor-extraction protocol. No response body text is retained, and these facts state what our probe received under an actor’s documented identity — never what that provider’s systems receive.", "Tracked AI-usage header values are parsed under a fixed, closed opt-out token grammar; only the parsed tokens, the agent-scoped map and an unparseable count are recorded — never the raw header string (values are truncated to 512 characters before parsing and anything off-grammar is only counted).", "Observed from Generative Metrics’ own network vantage with a single unauthenticated GET carrying the actor’s vendor-documented User-Agent; it does not prove what the provider’s systems experience or do." ] }, { "protocolExecutionId": "63ff1798-56ec-428d-b0df-de21d2f0c325", "protocolId": "access.actor-probe", "protocolVersion": "1.0.0", "stage": "ACQUISITION", "actor": { "actorId": "gm:control-probe", "provider": "Generative Metrics", "displayName": "Generative Metrics control probe", "productToken": null, "purpose": "UNSPECIFIED", "robotsPolicyApplicability": "NOT_APPLICABLE" }, "required": false, "input": { "requestedUrl": "https://www.usa.gov/" }, "startedAt": "2026-10-23T12:23:17.755Z", "finishedAt": "2026-10-23T12:23:17.755Z", "artifactRefs": [], "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "dd154c7d-5595-4d99-bc25-f08333f56950", "protocolExecutionId": "63ff1798-56ec-428d-b0df-de21d2f0c325", "kind": "ACTOR_ACCESS_RESULT", "result": "OBSERVED", "evidenceClass": "DIRECT", "value": { "actorId": "gm:control-probe", "requestUserAgent": null, "httpStatus": 200, "classification": "REACHED", "responseByteLength": 48647, "bodyTruncated": false, "failureCause": null, "finalUrl": "https://www.usa.gov/", "controlProbe": true }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:12.598Z", "limitations": [ "Probe results are single-request HTTP observations taken at scan time from Generative Metrics’ own servers; they do not observe or predict what any provider’s systems actually experience or do.", "Classifications are categorical execution states derived from the HTTP status alone; they are never scores. Enforcement for required actors is also recorded as sealed access.actor-probe evidence, where an uncorroborated denial is recorded as indeterminate (WARNING) and a robots.txt policy denial remains the deterministic production blocker.", "Each probe carries the actor’s vendor-documented canonical User-Agent; where a vendor publishes no fixed User-Agent string, the documented product token is sent instead and the fallback is recorded on the probe row (userAgentSource).", "Bounded content facts (machine-extracted word count, one hash of the extracted text, bounded flags) are computed from the same probe response bodies and are compared with the scanner’s own fetch by the content.actor-extraction protocol. No response body text is retained, and these facts state what our probe received under an actor’s documented identity — never what that provider’s systems receive.", "Tracked AI-usage header values are parsed under a fixed, closed opt-out token grammar; only the parsed tokens, the agent-scoped map and an unparseable count are recorded — never the raw header string (values are truncated to 512 characters before parsing and anything off-grammar is only counted).", "CONTROL_ROBOTS_GOVERNANCE: convention — the control request obeys robots.txt for our published scanner token; our own crawling rules are judged from the crawler preflight’s captured policy before anything is sent, and a policy disallow skips the knock without traffic (control_not_attempted: robots_disallow).", "GENERATIVE_METRICS_CONTROL_PROBE: measurement only — one plain request as the scanner’s published identity, sent to contrast AI-actor-specific refusals with refusals that extend to ordinary robot traffic. It derives no finding of its own, and intent is never inferred; from 4.0.0 its answer feeds the within-phase contrast test, and our own scan traffic immediately before it may have induced a block." ] } ], "limitations": [ "Probe results are single-request HTTP observations taken at scan time from Generative Metrics’ own servers; they do not observe or predict what any provider’s systems actually experience or do.", "Classifications are categorical execution states derived from the HTTP status alone; they are never scores. Enforcement for required actors is also recorded as sealed access.actor-probe evidence, where an uncorroborated denial is recorded as indeterminate (WARNING) and a robots.txt policy denial remains the deterministic production blocker.", "Each probe carries the actor’s vendor-documented canonical User-Agent; where a vendor publishes no fixed User-Agent string, the documented product token is sent instead and the fallback is recorded on the probe row (userAgentSource).", "Bounded content facts (machine-extracted word count, one hash of the extracted text, bounded flags) are computed from the same probe response bodies and are compared with the scanner’s own fetch by the content.actor-extraction protocol. No response body text is retained, and these facts state what our probe received under an actor’s documented identity — never what that provider’s systems receive.", "Tracked AI-usage header values are parsed under a fixed, closed opt-out token grammar; only the parsed tokens, the agent-scoped map and an unparseable count are recorded — never the raw header string (values are truncated to 512 characters before parsing and anything off-grammar is only counted).", "CONTROL_ROBOTS_GOVERNANCE: convention — the control request obeys robots.txt for our published scanner token; our own crawling rules are judged from the crawler preflight’s captured policy before anything is sent, and a policy disallow skips the knock without traffic (control_not_attempted: robots_disallow).", "GENERATIVE_METRICS_CONTROL_PROBE: measurement only — one plain request as the scanner’s published identity, sent to contrast AI-actor-specific refusals with refusals that extend to ordinary robot traffic. It derives no finding of its own, and intent is never inferred; from 4.0.0 its answer feeds the within-phase contrast test, and our own scan traffic immediately before it may have induced a block." ] }, { "protocolExecutionId": "bf95a686-e142-4951-acf8-383f59d3b0f4", "protocolId": "content.actor-extraction", "protocolVersion": "1.0.0", "stage": "EXTRACTION", "required": false, "actor": { "actorId": "gm:actor-content-probe", "provider": "Generative Metrics", "displayName": "Generative Metrics per-actor content probe", "purpose": "UNSPECIFIED", "robotsPolicyApplicability": "NOT_APPLICABLE", "userAgent": "GenerativeMetrics/1.0 (+https://generativemetrics.com)" }, "startedAt": "2026-10-23T12:23:18.116Z", "finishedAt": "2026-10-23T12:23:18.116Z", "artifactRefs": [], "input": { "requestedUrl": "https://www.usa.gov/", "comparisonBaselineSource": "GENERATIVE_METRICS_SCANNER_DIRECT_FETCH", "probeVantage": "GENERATIVE_METRICS_PROBE_REQUEST", "actorsCompared": 12, "actorsWithCollectedFacts": 12, "minComparableWords": 100, "materialDivergenceRatio": 0.5 }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "93ae1aeb-129c-4847-a833-891fc8280948", "protocolExecutionId": "bf95a686-e142-4951-acf8-383f59d3b0f4", "kind": "ACTOR_CONTENT_EXTRACTION", "result": "OBSERVED", "evidenceClass": "DERIVED", "value": { "actorId": "openai:oai-searchbot", "actorProvider": "OpenAI", "productToken": "OAI-SearchBot", "probeVantage": "GENERATIVE_METRICS_PROBE_REQUEST", "providerSystemsTookPart": false, "statesWhatProviderReceives": false, "comparisonBaselineSource": "GENERATIVE_METRICS_SCANNER_DIRECT_FETCH", "httpStatus": 200, "contentType": "text/html", "bodyTruncated": false, "contentFactsCollected": true, "extractionOutcome": "EXTRACTED", "actorWordCount": 443, "actorContentSha256": "db96ea54f61ce4103071357c19829db8244324c2a41ed2f9aaea48a52b3be884", "baselineWordCount": 443, "baselineContentSha256": "db96ea54f61ce4103071357c19829db8244324c2a41ed2f9aaea48a52b3be884", "comparisonAvailable": true, "contentComparisonReason": null, "wordCountRatio": 1, "materiallyDivergent": false, "wordCountDivergenceDirection": "NONE", "textSha256Differs": false, "accessGateSignal": null, "observedAt": "2026-10-23T12:23:00.409Z" }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:00.409Z", "limitations": [ "VANTAGE_LIMITATION: convention — this records only what Generative Metrics’ own probe request carrying the actor’s declared product token received from the audited URL at scan time. No provider system took part in this observation, and it does not state what that provider’s crawler, or any other system, receives.", "ACTOR_CONTENT_ACTOR_SCOPE: convention — the word count and hash recorded here are what the Generative Metrics probe received while carrying the OAI-SearchBot product token; they are not evidence about OpenAI’s systems and do not describe what that actor receives from the provider’s side." ] }, { "observationId": "8f96b91e-3567-40f1-b55d-ef19000e2057", "protocolExecutionId": "bf95a686-e142-4951-acf8-383f59d3b0f4", "kind": "ACTOR_CONTENT_EXTRACTION", "result": "OBSERVED", "evidenceClass": "DERIVED", "value": { "actorId": "anthropic:claude-searchbot", "actorProvider": "Anthropic", "productToken": "Claude-SearchBot", "probeVantage": "GENERATIVE_METRICS_PROBE_REQUEST", "providerSystemsTookPart": false, "statesWhatProviderReceives": false, "comparisonBaselineSource": "GENERATIVE_METRICS_SCANNER_DIRECT_FETCH", "httpStatus": 200, "contentType": "text/html", "bodyTruncated": false, "contentFactsCollected": true, "extractionOutcome": "EXTRACTED", "actorWordCount": 443, "actorContentSha256": "db96ea54f61ce4103071357c19829db8244324c2a41ed2f9aaea48a52b3be884", "baselineWordCount": 443, "baselineContentSha256": "db96ea54f61ce4103071357c19829db8244324c2a41ed2f9aaea48a52b3be884", "comparisonAvailable": true, "contentComparisonReason": null, "wordCountRatio": 1, "materiallyDivergent": false, "wordCountDivergenceDirection": "NONE", "textSha256Differs": false, "accessGateSignal": null, "observedAt": "2026-10-23T12:23:01.340Z" }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:01.340Z", "limitations": [ "VANTAGE_LIMITATION: convention — this records only what Generative Metrics’ own probe request carrying the actor’s declared product token received from the audited URL at scan time. No provider system took part in this observation, and it does not state what that provider’s crawler, or any other system, receives.", "ACTOR_CONTENT_ACTOR_SCOPE: convention — the word count and hash recorded here are what the Generative Metrics probe received while carrying the Claude-SearchBot product token; they are not evidence about Anthropic’s systems and do not describe what that actor receives from the provider’s side." ] }, { "observationId": "61fbd190-de21-4efb-ab9c-f069c39d0b79", "protocolExecutionId": "bf95a686-e142-4951-acf8-383f59d3b0f4", "kind": "ACTOR_CONTENT_EXTRACTION", "result": "OBSERVED", "evidenceClass": "DERIVED", "value": { "actorId": "google:googlebot", "actorProvider": "Google", "productToken": "Googlebot", "probeVantage": "GENERATIVE_METRICS_PROBE_REQUEST", "providerSystemsTookPart": false, "statesWhatProviderReceives": false, "comparisonBaselineSource": "GENERATIVE_METRICS_SCANNER_DIRECT_FETCH", "httpStatus": 200, "contentType": "text/html", "bodyTruncated": false, "contentFactsCollected": true, "extractionOutcome": "EXTRACTED", "actorWordCount": 443, "actorContentSha256": "db96ea54f61ce4103071357c19829db8244324c2a41ed2f9aaea48a52b3be884", "baselineWordCount": 443, "baselineContentSha256": "db96ea54f61ce4103071357c19829db8244324c2a41ed2f9aaea48a52b3be884", "comparisonAvailable": true, "contentComparisonReason": null, "wordCountRatio": 1, "materiallyDivergent": false, "wordCountDivergenceDirection": "NONE", "textSha256Differs": false, "accessGateSignal": null, "observedAt": "2026-10-23T12:23:02.386Z" }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:02.386Z", "limitations": [ "VANTAGE_LIMITATION: convention — this records only what Generative Metrics’ own probe request carrying the actor’s declared product token received from the audited URL at scan time. No provider system took part in this observation, and it does not state what that provider’s crawler, or any other system, receives.", "ACTOR_CONTENT_ACTOR_SCOPE: convention — the word count and hash recorded here are what the Generative Metrics probe received while carrying the Googlebot product token; they are not evidence about Google’s systems and do not describe what that actor receives from the provider’s side." ] }, { "observationId": "b2406d71-978d-43d7-9ad9-f93069c8f982", "protocolExecutionId": "bf95a686-e142-4951-acf8-383f59d3b0f4", "kind": "ACTOR_CONTENT_EXTRACTION", "result": "OBSERVED", "evidenceClass": "DERIVED", "value": { "actorId": "perplexity:perplexitybot", "actorProvider": "Perplexity", "productToken": "PerplexityBot", "probeVantage": "GENERATIVE_METRICS_PROBE_REQUEST", "providerSystemsTookPart": false, "statesWhatProviderReceives": false, "comparisonBaselineSource": "GENERATIVE_METRICS_SCANNER_DIRECT_FETCH", "httpStatus": 200, "contentType": "text/html", "bodyTruncated": false, "contentFactsCollected": true, "extractionOutcome": "EXTRACTED", "actorWordCount": 443, "actorContentSha256": "db96ea54f61ce4103071357c19829db8244324c2a41ed2f9aaea48a52b3be884", "baselineWordCount": 443, "baselineContentSha256": "db96ea54f61ce4103071357c19829db8244324c2a41ed2f9aaea48a52b3be884", "comparisonAvailable": true, "contentComparisonReason": null, "wordCountRatio": 1, "materiallyDivergent": false, "wordCountDivergenceDirection": "NONE", "textSha256Differs": false, "accessGateSignal": null, "observedAt": "2026-10-23T12:23:03.352Z" }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:03.352Z", "limitations": [ "VANTAGE_LIMITATION: convention — this records only what Generative Metrics’ own probe request carrying the actor’s declared product token received from the audited URL at scan time. No provider system took part in this observation, and it does not state what that provider’s crawler, or any other system, receives.", "ACTOR_CONTENT_ACTOR_SCOPE: convention — the word count and hash recorded here are what the Generative Metrics probe received while carrying the PerplexityBot product token; they are not evidence about Perplexity’s systems and do not describe what that actor receives from the provider’s side." ] }, { "observationId": "a091d91c-8f59-4fde-9d85-a7c4898f3969", "protocolExecutionId": "bf95a686-e142-4951-acf8-383f59d3b0f4", "kind": "ACTOR_CONTENT_EXTRACTION", "result": "OBSERVED", "evidenceClass": "DERIVED", "value": { "actorId": "openai:gptbot", "actorProvider": "OpenAI", "productToken": "GPTBot", "probeVantage": "GENERATIVE_METRICS_PROBE_REQUEST", "providerSystemsTookPart": false, "statesWhatProviderReceives": false, "comparisonBaselineSource": "GENERATIVE_METRICS_SCANNER_DIRECT_FETCH", "httpStatus": 200, "contentType": "text/html", "bodyTruncated": false, "contentFactsCollected": true, "extractionOutcome": "EXTRACTED", "actorWordCount": 443, "actorContentSha256": "db96ea54f61ce4103071357c19829db8244324c2a41ed2f9aaea48a52b3be884", "baselineWordCount": 443, "baselineContentSha256": "db96ea54f61ce4103071357c19829db8244324c2a41ed2f9aaea48a52b3be884", "comparisonAvailable": true, "contentComparisonReason": null, "wordCountRatio": 1, "materiallyDivergent": false, "wordCountDivergenceDirection": "NONE", "textSha256Differs": false, "accessGateSignal": null, "observedAt": "2026-10-23T12:23:04.420Z" }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:04.420Z", "limitations": [ "VANTAGE_LIMITATION: convention — this records only what Generative Metrics’ own probe request carrying the actor’s declared product token received from the audited URL at scan time. No provider system took part in this observation, and it does not state what that provider’s crawler, or any other system, receives.", "ACTOR_CONTENT_ACTOR_SCOPE: convention — the word count and hash recorded here are what the Generative Metrics probe received while carrying the GPTBot product token; they are not evidence about OpenAI’s systems and do not describe what that actor receives from the provider’s side." ] }, { "observationId": "0d1c0885-7bce-4fc5-b6a7-496744558506", "protocolExecutionId": "bf95a686-e142-4951-acf8-383f59d3b0f4", "kind": "ACTOR_CONTENT_EXTRACTION", "result": "OBSERVED", "evidenceClass": "DERIVED", "value": { "actorId": "anthropic:claudebot", "actorProvider": "Anthropic", "productToken": "ClaudeBot", "probeVantage": "GENERATIVE_METRICS_PROBE_REQUEST", "providerSystemsTookPart": false, "statesWhatProviderReceives": false, "comparisonBaselineSource": "GENERATIVE_METRICS_SCANNER_DIRECT_FETCH", "httpStatus": 200, "contentType": "text/html", "bodyTruncated": false, "contentFactsCollected": true, "extractionOutcome": "EXTRACTED", "actorWordCount": 443, "actorContentSha256": "db96ea54f61ce4103071357c19829db8244324c2a41ed2f9aaea48a52b3be884", "baselineWordCount": 443, "baselineContentSha256": "db96ea54f61ce4103071357c19829db8244324c2a41ed2f9aaea48a52b3be884", "comparisonAvailable": true, "contentComparisonReason": null, "wordCountRatio": 1, "materiallyDivergent": false, "wordCountDivergenceDirection": "NONE", "textSha256Differs": false, "accessGateSignal": null, "observedAt": "2026-10-23T12:23:05.358Z" }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:05.358Z", "limitations": [ "VANTAGE_LIMITATION: convention — this records only what Generative Metrics’ own probe request carrying the actor’s declared product token received from the audited URL at scan time. No provider system took part in this observation, and it does not state what that provider’s crawler, or any other system, receives.", "ACTOR_CONTENT_ACTOR_SCOPE: convention — the word count and hash recorded here are what the Generative Metrics probe received while carrying the ClaudeBot product token; they are not evidence about Anthropic’s systems and do not describe what that actor receives from the provider’s side." ] }, { "observationId": "a2bec722-ba3a-4985-ba84-17e65ff06fb3", "protocolExecutionId": "bf95a686-e142-4951-acf8-383f59d3b0f4", "kind": "ACTOR_CONTENT_EXTRACTION", "result": "OBSERVED", "evidenceClass": "DERIVED", "value": { "actorId": "openai:chatgpt-user", "actorProvider": "OpenAI", "productToken": "ChatGPT-User", "probeVantage": "GENERATIVE_METRICS_PROBE_REQUEST", "providerSystemsTookPart": false, "statesWhatProviderReceives": false, "comparisonBaselineSource": "GENERATIVE_METRICS_SCANNER_DIRECT_FETCH", "httpStatus": 200, "contentType": "text/html", "bodyTruncated": false, "contentFactsCollected": true, "extractionOutcome": "EXTRACTED", "actorWordCount": 443, "actorContentSha256": "db96ea54f61ce4103071357c19829db8244324c2a41ed2f9aaea48a52b3be884", "baselineWordCount": 443, "baselineContentSha256": "db96ea54f61ce4103071357c19829db8244324c2a41ed2f9aaea48a52b3be884", "comparisonAvailable": true, "contentComparisonReason": null, "wordCountRatio": 1, "materiallyDivergent": false, "wordCountDivergenceDirection": "NONE", "textSha256Differs": false, "accessGateSignal": null, "observedAt": "2026-10-23T12:23:06.351Z" }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:06.351Z", "limitations": [ "VANTAGE_LIMITATION: convention — this records only what Generative Metrics’ own probe request carrying the actor’s declared product token received from the audited URL at scan time. No provider system took part in this observation, and it does not state what that provider’s crawler, or any other system, receives.", "ACTOR_CONTENT_ACTOR_SCOPE: convention — the word count and hash recorded here are what the Generative Metrics probe received while carrying the ChatGPT-User product token; they are not evidence about OpenAI’s systems and do not describe what that actor receives from the provider’s side." ] }, { "observationId": "f021acbd-a0c3-446e-a3d8-d877420fe743", "protocolExecutionId": "bf95a686-e142-4951-acf8-383f59d3b0f4", "kind": "ACTOR_CONTENT_EXTRACTION", "result": "OBSERVED", "evidenceClass": "DERIVED", "value": { "actorId": "anthropic:claude-user", "actorProvider": "Anthropic", "productToken": "Claude-User", "probeVantage": "GENERATIVE_METRICS_PROBE_REQUEST", "providerSystemsTookPart": false, "statesWhatProviderReceives": false, "comparisonBaselineSource": "GENERATIVE_METRICS_SCANNER_DIRECT_FETCH", "httpStatus": 200, "contentType": "text/html", "bodyTruncated": false, "contentFactsCollected": true, "extractionOutcome": "EXTRACTED", "actorWordCount": 443, "actorContentSha256": "db96ea54f61ce4103071357c19829db8244324c2a41ed2f9aaea48a52b3be884", "baselineWordCount": 443, "baselineContentSha256": "db96ea54f61ce4103071357c19829db8244324c2a41ed2f9aaea48a52b3be884", "comparisonAvailable": true, "contentComparisonReason": null, "wordCountRatio": 1, "materiallyDivergent": false, "wordCountDivergenceDirection": "NONE", "textSha256Differs": false, "accessGateSignal": null, "observedAt": "2026-10-23T12:23:07.426Z" }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:07.426Z", "limitations": [ "VANTAGE_LIMITATION: convention — this records only what Generative Metrics’ own probe request carrying the actor’s declared product token received from the audited URL at scan time. No provider system took part in this observation, and it does not state what that provider’s crawler, or any other system, receives.", "ACTOR_CONTENT_ACTOR_SCOPE: convention — the word count and hash recorded here are what the Generative Metrics probe received while carrying the Claude-User product token; they are not evidence about Anthropic’s systems and do not describe what that actor receives from the provider’s side." ] }, { "observationId": "3bcd4f1a-7afc-4f6c-b96a-fbf3ebf8c6dc", "protocolExecutionId": "bf95a686-e142-4951-acf8-383f59d3b0f4", "kind": "ACTOR_CONTENT_EXTRACTION", "result": "OBSERVED", "evidenceClass": "DERIVED", "value": { "actorId": "perplexity:perplexity-user", "actorProvider": "Perplexity", "productToken": "Perplexity-User", "probeVantage": "GENERATIVE_METRICS_PROBE_REQUEST", "providerSystemsTookPart": false, "statesWhatProviderReceives": false, "comparisonBaselineSource": "GENERATIVE_METRICS_SCANNER_DIRECT_FETCH", "httpStatus": 200, "contentType": "text/html", "bodyTruncated": false, "contentFactsCollected": true, "extractionOutcome": "EXTRACTED", "actorWordCount": 443, "actorContentSha256": "db96ea54f61ce4103071357c19829db8244324c2a41ed2f9aaea48a52b3be884", "baselineWordCount": 443, "baselineContentSha256": "db96ea54f61ce4103071357c19829db8244324c2a41ed2f9aaea48a52b3be884", "comparisonAvailable": true, "contentComparisonReason": null, "wordCountRatio": 1, "materiallyDivergent": false, "wordCountDivergenceDirection": "NONE", "textSha256Differs": false, "accessGateSignal": null, "observedAt": "2026-10-23T12:23:08.382Z" }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:08.382Z", "limitations": [ "VANTAGE_LIMITATION: convention — this records only what Generative Metrics’ own probe request carrying the actor’s declared product token received from the audited URL at scan time. No provider system took part in this observation, and it does not state what that provider’s crawler, or any other system, receives.", "ACTOR_CONTENT_ACTOR_SCOPE: convention — the word count and hash recorded here are what the Generative Metrics probe received while carrying the Perplexity-User product token; they are not evidence about Perplexity’s systems and do not describe what that actor receives from the provider’s side." ] }, { "observationId": "f8c27ad3-e5dd-4402-b9f1-e4c954fca85d", "protocolExecutionId": "bf95a686-e142-4951-acf8-383f59d3b0f4", "kind": "ACTOR_CONTENT_EXTRACTION", "result": "OBSERVED", "evidenceClass": "DERIVED", "value": { "actorId": "google:google-agent", "actorProvider": "Google", "productToken": "Google-Agent", "probeVantage": "GENERATIVE_METRICS_PROBE_REQUEST", "providerSystemsTookPart": false, "statesWhatProviderReceives": false, "comparisonBaselineSource": "GENERATIVE_METRICS_SCANNER_DIRECT_FETCH", "httpStatus": 200, "contentType": "text/html", "bodyTruncated": false, "contentFactsCollected": true, "extractionOutcome": "EXTRACTED", "actorWordCount": 443, "actorContentSha256": "db96ea54f61ce4103071357c19829db8244324c2a41ed2f9aaea48a52b3be884", "baselineWordCount": 443, "baselineContentSha256": "db96ea54f61ce4103071357c19829db8244324c2a41ed2f9aaea48a52b3be884", "comparisonAvailable": true, "contentComparisonReason": null, "wordCountRatio": 1, "materiallyDivergent": false, "wordCountDivergenceDirection": "NONE", "textSha256Differs": false, "accessGateSignal": null, "observedAt": "2026-10-23T12:23:09.367Z" }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:09.367Z", "limitations": [ "VANTAGE_LIMITATION: convention — this records only what Generative Metrics’ own probe request carrying the actor’s declared product token received from the audited URL at scan time. No provider system took part in this observation, and it does not state what that provider’s crawler, or any other system, receives.", "ACTOR_CONTENT_ACTOR_SCOPE: convention — the word count and hash recorded here are what the Generative Metrics probe received while carrying the Google-Agent product token; they are not evidence about Google’s systems and do not describe what that actor receives from the provider’s side." ] }, { "observationId": "5484870a-4191-4e1a-bac1-7eeb04e1bbd5", "protocolExecutionId": "bf95a686-e142-4951-acf8-383f59d3b0f4", "kind": "ACTOR_CONTENT_EXTRACTION", "result": "OBSERVED", "evidenceClass": "DERIVED", "value": { "actorId": "openai:oai-adsbot", "actorProvider": "OpenAI", "productToken": "OAI-AdsBot", "probeVantage": "GENERATIVE_METRICS_PROBE_REQUEST", "providerSystemsTookPart": false, "statesWhatProviderReceives": false, "comparisonBaselineSource": "GENERATIVE_METRICS_SCANNER_DIRECT_FETCH", "httpStatus": 200, "contentType": "text/html", "bodyTruncated": false, "contentFactsCollected": true, "extractionOutcome": "EXTRACTED", "actorWordCount": 443, "actorContentSha256": "db96ea54f61ce4103071357c19829db8244324c2a41ed2f9aaea48a52b3be884", "baselineWordCount": 443, "baselineContentSha256": "db96ea54f61ce4103071357c19829db8244324c2a41ed2f9aaea48a52b3be884", "comparisonAvailable": true, "contentComparisonReason": null, "wordCountRatio": 1, "materiallyDivergent": false, "wordCountDivergenceDirection": "NONE", "textSha256Differs": false, "accessGateSignal": null, "observedAt": "2026-10-23T12:23:10.439Z" }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:10.439Z", "limitations": [ "VANTAGE_LIMITATION: convention — this records only what Generative Metrics’ own probe request carrying the actor’s declared product token received from the audited URL at scan time. No provider system took part in this observation, and it does not state what that provider’s crawler, or any other system, receives.", "ACTOR_CONTENT_ACTOR_SCOPE: convention — the word count and hash recorded here are what the Generative Metrics probe received while carrying the OAI-AdsBot product token; they are not evidence about OpenAI’s systems and do not describe what that actor receives from the provider’s side." ] }, { "observationId": "fd229d9b-68c2-43c9-a7e2-3a2bb4a26bd9", "protocolExecutionId": "bf95a686-e142-4951-acf8-383f59d3b0f4", "kind": "ACTOR_CONTENT_EXTRACTION", "result": "OBSERVED", "evidenceClass": "DERIVED", "value": { "actorId": "kagi:kagibot", "actorProvider": "Kagi", "productToken": "Kagibot", "probeVantage": "GENERATIVE_METRICS_PROBE_REQUEST", "providerSystemsTookPart": false, "statesWhatProviderReceives": false, "comparisonBaselineSource": "GENERATIVE_METRICS_SCANNER_DIRECT_FETCH", "httpStatus": 200, "contentType": "text/html", "bodyTruncated": false, "contentFactsCollected": true, "extractionOutcome": "EXTRACTED", "actorWordCount": 443, "actorContentSha256": "db96ea54f61ce4103071357c19829db8244324c2a41ed2f9aaea48a52b3be884", "baselineWordCount": 443, "baselineContentSha256": "db96ea54f61ce4103071357c19829db8244324c2a41ed2f9aaea48a52b3be884", "comparisonAvailable": true, "contentComparisonReason": null, "wordCountRatio": 1, "materiallyDivergent": false, "wordCountDivergenceDirection": "NONE", "textSha256Differs": false, "accessGateSignal": null, "observedAt": "2026-10-23T12:23:11.418Z" }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:11.418Z", "limitations": [ "VANTAGE_LIMITATION: convention — this records only what Generative Metrics’ own probe request carrying the actor’s declared product token received from the audited URL at scan time. No provider system took part in this observation, and it does not state what that provider’s crawler, or any other system, receives.", "ACTOR_CONTENT_ACTOR_SCOPE: convention — the word count and hash recorded here are what the Generative Metrics probe received while carrying the Kagibot product token; they are not evidence about Kagi’s systems and do not describe what that actor receives from the provider’s side." ] } ], "limitations": [ "Probe results are single-request HTTP observations taken at scan time from Generative Metrics’ own servers; they do not observe or predict what any provider’s systems actually experience or do.", "Classifications are categorical execution states derived from the HTTP status alone; they are never scores. Enforcement for required actors is also recorded as sealed access.actor-probe evidence, where an uncorroborated denial is recorded as indeterminate (WARNING) and a robots.txt policy denial remains the deterministic production blocker.", "Probes deliberately do not consult robots.txt (the audit measures the declared-versus-enforced gap) and send no disclosure header (measurement neutrality).", "Each probe carries the actor’s vendor-documented canonical User-Agent; where a vendor publishes no fixed User-Agent string, the documented product token is sent instead and the fallback is recorded on the probe row (userAgentSource).", "Bounded content facts (machine-extracted word count, one hash of the extracted text, bounded flags) are computed from the same probe response bodies and are compared with the scanner’s own fetch by the content.actor-extraction protocol. No response body text is retained, and these facts state what our probe received under an actor’s documented identity — never what that provider’s systems receive.", "Tracked AI-usage header values are parsed under a fixed, closed opt-out token grammar; only the parsed tokens, the agent-scoped map and an unparseable count are recorded — never the raw header string (values are truncated to 512 characters before parsing and anything off-grammar is only counted).", "VANTAGE_LIMITATION: convention — this records only what Generative Metrics’ own probe request carrying the actor’s declared product token received from the audited URL at scan time. No provider system took part in this observation, and it does not state what that provider’s crawler, or any other system, receives.", "ACTOR_CONTENT_PROBE_SCOPE: convention — a request carrying a documented product token is not that provider’s system: our single probe’s address, headers, timing and cadence differ from the provider’s own fetchers, so this states what OUR probe received under that name and never what the provider receives.", "ACTOR_CONTENT_DIVERGENCE_CONVENTION: convention — a divergence is reported when the smaller machine-extracted word count is below 0.5 of the larger and the larger count is at least 100 words. This mirrors the raw-versus-rendered main-content convention in this protocol set; it is a reporting convention, not a score, not a quality threshold and never a verdict input.", "ACTOR_CONTENT_EXTRACTED_ONLY: convention — both sides are measured with the same extractor (linkedom_readability/text_extraction_v3). Raw-body and whole-document hashes are never the signal, because nonces, timestamps, personalisation and cache variation change bytes without changing extractable content; extracted-text hashes are recorded for both sides and a hash difference alone is never reported.", "The probe request carries the actor’s product token as its user agent and no Accept header, while the scanner’s own fetch sends its own user agent and an Accept header; a divergence can also be caused by that difference, so it is reported for review rather than assigned to the actor name alone.", "Only HTML representations are compared; a probe response that declared another media type (for example markdown) is recorded as not comparable rather than as a divergence.", "Only machine-extracted word counts, extracted-text hashes and bounded flags are retained from the probe response; no response body text and no raw body hash leaves the probe phase.", "ACTOR_CONTENT_BASELINE_UNAVAILABLE: convention — the scanner’s own fetch produced no machine-extracted main-content text for this URL, so there was nothing to compare the per-actor content against." ] }, { "protocolExecutionId": "01b45a24-ee9e-4ba8-9800-077d2425b603", "protocolId": "extraction.main-content", "protocolVersion": "1.0.0", "stage": "EXTRACTION", "required": true, "actor": { "actorId": "gm:direct-http-probe", "provider": "Generative Metrics", "displayName": "Generative Metrics direct HTTP probe", "purpose": "UNSPECIFIED", "robotsPolicyApplicability": "NOT_APPLICABLE", "userAgent": "GenerativeMetrics/1.0 (+https://generativemetrics.com)" }, "startedAt": "2026-10-23T12:23:17.806Z", "finishedAt": "2026-10-23T12:23:17.806Z", "artifactRefs": [ "f0cb1e46-f740-49c5-8cbb-ab49da5046db", "554ed831-6d27-4c75-83a7-0eace8fc2eb8", "8c624ab2-9f36-4a02-a4e2-43f3d2fd5b9b" ], "input": { "rawArtifactId": "f0cb1e46-f740-49c5-8cbb-ab49da5046db", "renderedArtifactId": "554ed831-6d27-4c75-83a7-0eace8fc2eb8", "noJavaScriptArtifactId": "8c624ab2-9f36-4a02-a4e2-43f3d2fd5b9b" }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "03badaaf-8aec-4947-98ef-26b9ba3c2169", "protocolExecutionId": "01b45a24-ee9e-4ba8-9800-077d2425b603", "kind": "MAIN_CONTENT_COMPARISON", "result": "OBSERVED", "evidenceClass": "DERIVED", "value": { "raw": { "outcome": "EXTRACTED", "wordCount": 443, "byteLength": 7259, "contentSha256": "db96ea54f61ce4103071357c19829db8244324c2a41ed2f9aaea48a52b3be884", "excerpt": "Making government services easier to find USA.gov helps you locate and understand government benefits, programs, and information. How do I ... Check my voter registration Get or renew a passport Get housing help Find unclaimed money Jump to all topics and services Government benefits and financial assistance Discover government benefits that you may be eligible for and learn how to apply. Find ben…", "detectedLanguage": "en", "extractionMethod": "linkedom_readability", "extractionVersion": "text_extraction_v3", "scopeRule": "semantic-first scope convention: the extractor deterministically selects the first visible main element, then [role=\"main\"], then article, in document order, and records WHICH section of that kind it selected as its zero-based position among the page's candidates of the same kind, with hidden candidates counted in the numbering; within the selected scope, style, script, template and noscript elements are removed before measuring; word counts include text inside hidden containers within the selected scope; the visibility rule tests only the candidate element itself, so a candidate whose ancestor element is hidden is still selected and the observation records main_content_hidden_ancestor: true as data; a stricter visibility rule is planned; an empty semantic scope is recorded as empty rather than replaced by a guess; when both passes of a comparison record which main-content section they measured, the two are compared only when it is the same section, and otherwise the comparison is recorded as not possible rather than as a divergence; when no visible semantic candidate exists, footer, header, nav, aside and [role=\"contentinfo\"] subtrees are pruned, consent-dialog and CMP containers (a fixed disclosed list: dialog/aria-modal markup, the onetrust id family and named vendor containers) are pruned the same way, and @mozilla/readability deterministically selects the scope from the pruned document — a consent interstitial is never measured as the page; the recorded main content is the selected subtree's text, and no main content is identified when no subtree qualifies.", "scopeElement": "main", "scopeSelector": "main#main-content", "scopeReason": "semantic", "scopeIndex": 0, "scopeHiddenAncestor": false, "scopeWithinConsentContainer": false, "consentContainersPruned": 0, "scopeSha256": "98f37ab634522227fb701028a8a7057ce89deec0255b0e0222e59d37e044c5c0", "scopeBytes": 15243, "coverage": 0.1492, "coverageLow": false, "hasTitle": true, "hasDescription": true, "paragraphCount": 30, "listCount": 9, "tableCount": 0, "linkDensity": 0.3279, "boilerplateRatio": 0.6422 }, "rendered": { "outcome": "EXTRACTED", "wordCount": 443, "byteLength": 7259, "contentSha256": "db96ea54f61ce4103071357c19829db8244324c2a41ed2f9aaea48a52b3be884", "excerpt": "Making government services easier to find USA.gov helps you locate and understand government benefits, programs, and information. How do I ... Check my voter registration Get or renew a passport Get housing help Find unclaimed money Jump to all topics and services Government benefits and financial assistance Discover government benefits that you may be eligible for and learn how to apply. Find ben…", "detectedLanguage": "en", "extractionMethod": "linkedom_readability", "extractionVersion": "text_extraction_v3", "scopeRule": "semantic-first scope convention: the extractor deterministically selects the first visible main element, then [role=\"main\"], then article, in document order, and records WHICH section of that kind it selected as its zero-based position among the page's candidates of the same kind, with hidden candidates counted in the numbering; within the selected scope, style, script, template and noscript elements are removed before measuring; word counts include text inside hidden containers within the selected scope; the visibility rule tests only the candidate element itself, so a candidate whose ancestor element is hidden is still selected and the observation records main_content_hidden_ancestor: true as data; a stricter visibility rule is planned; an empty semantic scope is recorded as empty rather than replaced by a guess; when both passes of a comparison record which main-content section they measured, the two are compared only when it is the same section, and otherwise the comparison is recorded as not possible rather than as a divergence; when no visible semantic candidate exists, footer, header, nav, aside and [role=\"contentinfo\"] subtrees are pruned, consent-dialog and CMP containers (a fixed disclosed list: dialog/aria-modal markup, the onetrust id family and named vendor containers) are pruned the same way, and @mozilla/readability deterministically selects the scope from the pruned document — a consent interstitial is never measured as the page; the recorded main content is the selected subtree's text, and no main content is identified when no subtree qualifies.", "scopeElement": "main", "scopeSelector": "main#main-content", "scopeReason": "semantic", "scopeIndex": 0, "scopeHiddenAncestor": false, "scopeWithinConsentContainer": false, "consentContainersPruned": 0, "scopeSha256": "41358745144e8b2de189ae0a853463ec5ec67a747fcaa8255c32ab5c4dc75ff9", "scopeBytes": 17395, "coverage": 0.134, "coverageLow": false, "hasTitle": true, "hasDescription": true, "paragraphCount": 30, "listCount": 9, "tableCount": 0, "linkDensity": 0.275, "boilerplateRatio": 0.6579 }, "noJavaScript": { "outcome": "EXTRACTED", "wordCount": 443, "byteLength": 7259, "contentSha256": "db96ea54f61ce4103071357c19829db8244324c2a41ed2f9aaea48a52b3be884", "excerpt": "Making government services easier to find USA.gov helps you locate and understand government benefits, programs, and information. How do I ... Check my voter registration Get or renew a passport Get housing help Find unclaimed money Jump to all topics and services Government benefits and financial assistance Discover government benefits that you may be eligible for and learn how to apply. Find ben…", "detectedLanguage": "en", "extractionMethod": "linkedom_readability", "extractionVersion": "text_extraction_v3", "scopeRule": "semantic-first scope convention: the extractor deterministically selects the first visible main element, then [role=\"main\"], then article, in document order, and records WHICH section of that kind it selected as its zero-based position among the page's candidates of the same kind, with hidden candidates counted in the numbering; within the selected scope, style, script, template and noscript elements are removed before measuring; word counts include text inside hidden containers within the selected scope; the visibility rule tests only the candidate element itself, so a candidate whose ancestor element is hidden is still selected and the observation records main_content_hidden_ancestor: true as data; a stricter visibility rule is planned; an empty semantic scope is recorded as empty rather than replaced by a guess; when both passes of a comparison record which main-content section they measured, the two are compared only when it is the same section, and otherwise the comparison is recorded as not possible rather than as a divergence; when no visible semantic candidate exists, footer, header, nav, aside and [role=\"contentinfo\"] subtrees are pruned, consent-dialog and CMP containers (a fixed disclosed list: dialog/aria-modal markup, the onetrust id family and named vendor containers) are pruned the same way, and @mozilla/readability deterministically selects the scope from the pruned document — a consent interstitial is never measured as the page; the recorded main content is the selected subtree's text, and no main content is identified when no subtree qualifies.", "scopeElement": "main", "scopeSelector": "main#main-content", "scopeReason": "semantic", "scopeIndex": 0, "scopeHiddenAncestor": false, "scopeWithinConsentContainer": false, "consentContainersPruned": 0, "scopeSha256": "98f37ab634522227fb701028a8a7057ce89deec0255b0e0222e59d37e044c5c0", "scopeBytes": 15243, "coverage": 0.149, "coverageLow": false, "hasTitle": true, "hasDescription": true, "paragraphCount": 30, "listCount": 9, "tableCount": 0, "linkDensity": 0.3279, "boilerplateRatio": 0.6422 }, "rawToRenderedWordRatio": 1, "renderedToNoJavaScriptWordRatio": 1, "rawWordDeficit": 0, "containmentPct": 100, "anchorResidual": { "surplusWordsTotal": 0, "suppressedWords": 0, "novelWordsRemaining": 0, "hasNovelContentRemaining": false, "disclosure": "", "residualHonestyClause": "the raw corpus is extracted from standard block elements; surplus blocks that share no script-invariant anchors are indistinguishable from novel content under this convention and are recorded as a deficit; this convention is fixed and never tuned per scan." }, "langTokenPartition": "rendered 1373 tokens by leaf-most lang attribute (each token counted once): en 1373", "structure": { "blockCensus": { "value": null, "reason": "capture_no_scroll_pass" }, "regionOccupancy": { "value": null, "reason": "capture_no_scroll_pass" }, "deltaMajorityCjk": false } }, "artifactRefs": [ "f0cb1e46-f740-49c5-8cbb-ab49da5046db", "554ed831-6d27-4c75-83a7-0eace8fc2eb8", "8c624ab2-9f36-4a02-a4e2-43f3d2fd5b9b" ], "producedAt": "2026-10-23T12:23:17.806Z", "limitations": [ "Main-content extraction uses a deterministic Readability implementation and does not prove extraction by a downstream AI system.", "Readability scores are intentionally excluded from this protocol because language support is not universal.", "verbatim-repeated blocks (p, h1–h6, li, blockquote, pre, table cells, definition items) are counted once; translated variants are not deduplicated — different strings are different words, and the record says so.", "main-content convention: word counts include text inside hidden containers within the selected scope; the visibility rule tests only the candidate element itself, so a candidate whose ancestor element is hidden is still selected and the observation records main_content_hidden_ancestor: true as data; a stricter visibility rule is planned.", "acquisition provenance: the raw arm is fetched by our direct non-JavaScript HTTP fetcher and its request User-Agent is recorded on the raw artifact; the rendered and no-JavaScript arms are headless-browser captures whose request User-Agent is not separately recorded on those artifacts; per-actor probe requests record their request User-Agent on each probe observation. User-agent conditioning of content is not separately attributed between arms.", "containment convention: a fixed disclosed 90% — the same threshold on every site and every scan; containment describes the relationship between the rendered and no-JavaScript extractions, not intent." ] } ], "limitations": [ "Main-content extraction uses a deterministic Readability implementation and does not prove extraction by a downstream AI system.", "Readability scores are intentionally excluded from this protocol because language support is not universal.", "verbatim-repeated blocks (p, h1–h6, li, blockquote, pre, table cells, definition items) are counted once; translated variants are not deduplicated — different strings are different words, and the record says so.", "main-content convention: word counts include text inside hidden containers within the selected scope; the visibility rule tests only the candidate element itself, so a candidate whose ancestor element is hidden is still selected and the observation records main_content_hidden_ancestor: true as data; a stricter visibility rule is planned.", "acquisition provenance: the raw arm is fetched by our direct non-JavaScript HTTP fetcher and its request User-Agent is recorded on the raw artifact; the rendered and no-JavaScript arms are headless-browser captures whose request User-Agent is not separately recorded on those artifacts; per-actor probe requests record their request User-Agent on each probe observation. User-agent conditioning of content is not separately attributed between arms.", "containment convention: a fixed disclosed 90% — the same threshold on every site and every scan; containment describes the relationship between the rendered and no-JavaScript extractions, not intent." ] }, { "protocolExecutionId": "a7328952-d662-4c3b-a276-ccca447e0ace", "protocolId": "structured-data.visible-consistency", "protocolVersion": "1.0.0", "stage": "INTERPRETATION", "required": true, "actor": { "actorId": "gm:direct-http-probe", "provider": "Generative Metrics", "displayName": "Generative Metrics direct HTTP probe", "purpose": "UNSPECIFIED", "robotsPolicyApplicability": "NOT_APPLICABLE", "userAgent": "GenerativeMetrics/1.0 (+https://generativemetrics.com)" }, "startedAt": "2026-10-23T12:23:18.076Z", "finishedAt": "2026-10-23T12:23:18.076Z", "artifactRefs": [ "f0cb1e46-f740-49c5-8cbb-ab49da5046db", "554ed831-6d27-4c75-83a7-0eace8fc2eb8" ], "input": { "rawArtifactId": "f0cb1e46-f740-49c5-8cbb-ab49da5046db", "renderedArtifactId": "554ed831-6d27-4c75-83a7-0eace8fc2eb8", "materialProperties": [ "description", "headline", "name" ], "maxPropertyChecks": 100 }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "8ce63683-3aff-434d-9e45-8a168e99ebd5", "protocolExecutionId": "a7328952-d662-4c3b-a276-ccca447e0ace", "kind": "STRUCTURED_DATA_COMPARISON", "result": "OBSERVED", "evidenceClass": "DERIVED", "value": { "raw": { "diagnostics": { "jsonLdBlocksFound": 2, "jsonLdBlocksParsed": 2, "jsonLdBlocksFailed": 0, "jsonLdObjectsExtracted": 2, "microdataObjectsExtracted": 0, "rdfaObjectsExtracted": 0, "ogObjectsExtracted": 0, "fatalParserError": false }, "schemaCount": 2, "schemaTypes": [ "Organization", "WebSite" ] }, "rendered": { "diagnostics": { "jsonLdBlocksFound": 2, "jsonLdBlocksParsed": 2, "jsonLdBlocksFailed": 0, "jsonLdObjectsExtracted": 2, "microdataObjectsExtracted": 0, "rdfaObjectsExtracted": 0, "ogObjectsExtracted": 0, "fatalParserError": false }, "schemaCount": 2, "schemaTypes": [ "Organization", "WebSite" ] }, "schemaOnlyAfterBrowserRendering": false, "materialSchemaTypes": [ "Article", "NewsArticle", "BlogPosting", "Product", "Organization", "LocalBusiness", "FAQPage", "HowTo", "BreadcrumbList", "WebSite", "Offer", "Review", "Event", "Recipe" ], "excludedLandmarkSchemaTypes": [ "WPHeader", "WPFooter", "SiteNavigationElement", "WebPageElement" ], "materialTypesAfterRender": [ "Organization", "WebSite" ], "openGraphSynthesizedSchemasExcludedFromVisibleTextChecks": 0, "materialPropertyChecks": [ { "path": "$[0].name", "property": "name", "valueSha256": "f5d02fe52f836ca5615a2f3dc17f473ad1b0579ca5618cefc21c3f0d58bc91a9", "visibleMatch": true } ], "unmatchedMaterialPropertyCount": 0, "propertyChecksTruncated": false }, "artifactRefs": [ "f0cb1e46-f740-49c5-8cbb-ab49da5046db", "554ed831-6d27-4c75-83a7-0eace8fc2eb8" ], "producedAt": "2026-10-23T12:23:18.076Z", "limitations": [ "Visible consistency checks cover bounded name, headline, and description string properties only.", "A visible string match does not prove semantic correctness or downstream use.", "The absence of structured data is not automatically a defect.", "Structured-data objects synthesized from OpenGraph meta tags are excluded from visible-text checks; a meta description is metadata, not body copy.", "Materiality for the raw structured-data deficit decision uses a fixed disclosed list of schema types — the same list on every site and every scan: Article, NewsArticle, BlogPosting, Product, Organization, LocalBusiness, FAQPage, HowTo, BreadcrumbList, WebSite, Offer, Review, Event, Recipe. Landmark roles (WPHeader, WPFooter, SiteNavigationElement, WebPageElement) are excluded from materiality." ] } ], "limitations": [ "Visible consistency checks cover bounded name, headline, and description string properties only.", "A visible string match does not prove semantic correctness or downstream use.", "The absence of structured data is not automatically a defect.", "Structured-data objects synthesized from OpenGraph meta tags are excluded from visible-text checks; a meta description is metadata, not body copy.", "Materiality for the raw structured-data deficit decision uses a fixed disclosed list of schema types — the same list on every site and every scan: Article, NewsArticle, BlogPosting, Product, Organization, LocalBusiness, FAQPage, HowTo, BreadcrumbList, WebSite, Offer, Review, Event, Recipe. Landmark roles (WPHeader, WPFooter, SiteNavigationElement, WebPageElement) are excluded from materiality." ] }, { "protocolExecutionId": "0bc319bd-5095-428a-a132-e09d78243f6e", "protocolId": "structured-data.completeness", "protocolVersion": "1.0.0", "stage": "INTERPRETATION", "required": false, "actor": { "actorId": "gm:direct-http-probe", "provider": "Generative Metrics", "displayName": "Generative Metrics direct HTTP probe", "purpose": "UNSPECIFIED", "robotsPolicyApplicability": "NOT_APPLICABLE", "userAgent": "GenerativeMetrics/1.0 (+https://generativemetrics.com)" }, "startedAt": "2026-10-23T12:23:18.096Z", "finishedAt": "2026-10-23T12:23:18.096Z", "artifactRefs": [ "f0cb1e46-f740-49c5-8cbb-ab49da5046db", "554ed831-6d27-4c75-83a7-0eace8fc2eb8" ], "input": { "rawArtifactId": "f0cb1e46-f740-49c5-8cbb-ab49da5046db", "renderedArtifactId": "554ed831-6d27-4c75-83a7-0eace8fc2eb8", "maxSchemaTypes": 100, "organizationFields": [ "name", "url", "logo" ] }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "63c8adfd-f2c0-4cd4-81c8-708d04993542", "protocolExecutionId": "0bc319bd-5095-428a-a132-e09d78243f6e", "kind": "STRUCTURED_DATA_COMPLETENESS", "result": "OBSERVED", "evidenceClass": "DERIVED", "value": { "rawTypes": [ "Organization", "WebSite" ], "renderedTypesOnly": [], "organizationIdentity": { "present": true, "fields": { "name": false, "url": true, "logo": true }, "missingFields": [ "name" ] } }, "artifactRefs": [ "f0cb1e46-f740-49c5-8cbb-ab49da5046db", "554ed831-6d27-4c75-83a7-0eace8fc2eb8" ], "producedAt": "2026-10-23T12:23:18.096Z", "limitations": [ "Completeness checks cover raw-versus-rendered schema type presence and three bounded Organization identity fields only.", "Organization identity fields are recorded as booleans; property values are not stored.", "The absence of an Organization node is not automatically a defect." ] } ], "limitations": [ "Completeness checks cover raw-versus-rendered schema type presence and three bounded Organization identity fields only.", "Organization identity fields are recorded as booleans; property values are not stored.", "The absence of an Organization node is not automatically a defect." ] }, { "protocolExecutionId": "9e45bb55-b2df-4494-a640-be3a9de19ae0", "protocolId": "freshness.dates", "protocolVersion": "1.0.0", "stage": "INTERPRETATION", "required": true, "actor": { "actorId": "gm:direct-http-probe", "provider": "Generative Metrics", "displayName": "Generative Metrics direct HTTP probe", "purpose": "UNSPECIFIED", "robotsPolicyApplicability": "NOT_APPLICABLE", "userAgent": "GenerativeMetrics/1.0 (+https://generativemetrics.com)" }, "startedAt": "2026-10-23T12:23:18.106Z", "finishedAt": "2026-10-23T12:23:18.106Z", "artifactRefs": [ "f0cb1e46-f740-49c5-8cbb-ab49da5046db" ], "input": { "rawArtifactId": "f0cb1e46-f740-49c5-8cbb-ab49da5046db", "observedAt": "2026-10-23T12:23:13.857Z", "acceptedFormats": [ "YYYY-MM-DD", "ISO-8601-WITH-TIMEZONE", "IMF-FIXDATE" ] }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "57b118e8-a833-4b28-a85f-0ea6eea92085", "protocolExecutionId": "9e45bb55-b2df-4494-a640-be3a9de19ae0", "kind": "DATE_EVIDENCE", "result": "OBSERVED", "evidenceClass": "DERIVED", "value": { "status": "DATED", "validDates": [ { "kind": "MODIFIED", "source": "HTTP_HEADER", "path": "response.headers.last-modified", "normalizedUtc": "2026-10-22T19:46:15.000Z" } ], "rejectedDates": [], "newestDateUtc": "2026-10-22T19:46:15.000Z", "oldestDateUtc": "2026-10-22T19:46:15.000Z", "newestAgeDays": 0, "staleAgeDaysThreshold": 365, "stale": false, "contradiction": false }, "artifactRefs": [ "f0cb1e46-f740-49c5-8cbb-ab49da5046db" ], "producedAt": "2026-10-23T12:23:18.106Z", "limitations": [ "Free-form dates in visible prose are intentionally excluded from authoritative freshness evidence.", "Age is reported as evidence and is not converted into a universal stale/fresh score.", "STALE: convention — a newest supported date older than 365 days is labelled STALE for information only; the threshold is a Generative Metrics convention, not a published standard, and the label never changes the audit verdict." ] } ], "limitations": [ "Free-form dates in visible prose are intentionally excluded from authoritative freshness evidence.", "Age is reported as evidence and is not converted into a universal stale/fresh score.", "STALE: convention — a newest supported date older than 365 days is labelled STALE for information only; the threshold is a Generative Metrics convention, not a published standard, and the label never changes the audit verdict." ] }, { "protocolExecutionId": "b60ff6e7-c9dc-43d2-bfb2-eeca5d62c334", "protocolId": "identity.crawler-verification", "protocolVersion": "1.0.0", "stage": "VERIFICATION", "required": false, "actor": { "actorId": "gm:crawler-identity-verification", "provider": "Generative Metrics", "displayName": "Generative Metrics crawler identity verification", "purpose": "UNSPECIFIED", "robotsPolicyApplicability": "NOT_APPLICABLE", "userAgent": "GenerativeMetrics/1.0 (+https://generativemetrics.com)" }, "startedAt": "2026-10-23T12:23:18.117Z", "finishedAt": "2026-10-23T12:23:18.117Z", "artifactRefs": [], "input": { "requestedUrl": "https://www.usa.gov/", "verificationSource": "VENDOR_PUBLISHED_RANGES_VS_SYNTHETIC_PROBE", "verificationState": "RANGES_UNAVAILABLE", "verificationPerformed": false, "declaredProbeSourceEvidenceClass": "DECLARED", "declaredProbeSourceAddress": null, "declaredProbeSourceDeclaration": "GM_SCAN_EGRESS_ADDRESS_NOT_DECLARED", "rangeSourceDeclarations": [ { "sourceId": "openai:gptbot-json", "vendor": "OpenAI", "documentationUrl": "https://openai.com/gptbot.json", "coversActorIds": [ "openai:gptbot" ] }, { "sourceId": "openai:searchbot-json", "vendor": "OpenAI", "documentationUrl": "https://openai.com/searchbot.json", "coversActorIds": [ "openai:oai-searchbot" ] }, { "sourceId": "openai:chatgpt-user-json", "vendor": "OpenAI", "documentationUrl": "https://openai.com/chatgpt-user.json", "coversActorIds": [ "openai:chatgpt-user" ] }, { "sourceId": "openai:adsbot-json", "vendor": "OpenAI", "documentationUrl": "https://openai.com/adsbot.json", "coversActorIds": [ "openai:oai-adsbot" ] }, { "sourceId": "perplexity:perplexitybot-json", "vendor": "Perplexity", "documentationUrl": "https://www.perplexity.ai/perplexitybot.json", "coversActorIds": [ "perplexity:perplexitybot" ] }, { "sourceId": "anthropic:bots-json", "vendor": "Anthropic", "documentationUrl": "https://claude.com/crawling/bots.json", "coversActorIds": [ "anthropic:claudebot", "anthropic:claude-searchbot", "anthropic:claude-user" ] } ], "rangeDocumentsDeclared": 6, "rangeDocumentsRetrieved": 0, "rangeSnapshotRefreshedAt": null, "actorsAssessed": 27, "customerSideMeasurementInputAvailable": false }, "failure": null, "state": "NOT_APPLICABLE", "observations": [ { "observationId": "fa869f4a-b73c-4ef7-87a1-939b45146c20", "protocolExecutionId": "b60ff6e7-c9dc-43d2-bfb2-eeca5d62c334", "kind": "CRAWLER_IDENTITY_VERIFICATION", "result": "NOT_OBSERVED", "evidenceClass": "DECLARED", "value": { "actorId": "openai:oai-searchbot", "actorProvider": "OpenAI", "productToken": "OAI-SearchBot", "verificationState": "RANGES_UNAVAILABLE", "publishedRangesState": "RANGES_UNAVAILABLE", "verificationPerformed": false, "vendorSystemsTookPart": false, "canVerifyVendorCrawler": false, "customerSideMeasurementInputAvailable": false, "declaredSourceId": "openai:searchbot-json", "declaredSourceUrl": "https://openai.com/searchbot.json", "documentRetrieved": null, "documentShape": null, "documentHttpStatus": null, "documentByteLength": null, "documentBodyTruncated": null, "documentPayloadSha256": null, "documentRetrievedAt": null, "documentFailureCause": null, "publishedPrefixCount": null, "publishedIpv4PrefixCount": null, "publishedIpv6PrefixCount": null, "probeSourceAddress": null, "probeSourceEvidenceClass": "DECLARED" }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:18.117Z", "limitations": [ "VANTAGE_LIMITATION: convention — Generative Metrics observes from its own network vantage with a single synthetic request sent from its own declared scan address. A request from our address can never confirm that traffic claiming a crawler name is that vendor’s crawler, because the vendor’s fleet uses its own address ranges, reverse DNS, edge signatures and script behaviour. No verification result is claimed, in either direction, by this protocol.", "PROBE_SOURCE_DECLARED: convention — Generative Metrics’ own probe source address is a DECLARED self-report supplied as an input, never a measured observation, so it can never be read as evidence about a vendor’s crawler.", "VERIFICATION_NOT_PERFORMED: convention — this protocol records the state of the published identity material. It performs no match and no mismatch between a request and an address range, and its observation result is INDETERMINATE whenever a verification cannot be made. Every state reachable from Generative Metrics’ vantage is such a case.", "Only a payload hash, a bounded prefix count and a shape verdict are recorded for each vendor document; no document body and no address value leaves the retrieval phase.", "VANTAGE_LIMITED: convention — the vendor-published identity material is in hand for at least one audited actor, and the verification itself still cannot be performed from Generative Metrics’ vantage. The material is recorded so a site owner can use it directly with their own traffic measurements." ] }, { "observationId": "13e6e6f3-0d6b-4e0c-a56e-b3dfc94789e4", "protocolExecutionId": "b60ff6e7-c9dc-43d2-bfb2-eeca5d62c334", "kind": "CRAWLER_IDENTITY_VERIFICATION", "result": "NOT_OBSERVED", "evidenceClass": "DECLARED", "value": { "actorId": "anthropic:claude-searchbot", "actorProvider": "Anthropic", "productToken": "Claude-SearchBot", "verificationState": "RANGES_UNAVAILABLE", "publishedRangesState": "RANGES_UNAVAILABLE", "verificationPerformed": false, "vendorSystemsTookPart": false, "canVerifyVendorCrawler": false, "customerSideMeasurementInputAvailable": false, "declaredSourceId": "anthropic:bots-json", "declaredSourceUrl": "https://claude.com/crawling/bots.json", "documentRetrieved": null, "documentShape": null, "documentHttpStatus": null, "documentByteLength": null, "documentBodyTruncated": null, "documentPayloadSha256": null, "documentRetrievedAt": null, "documentFailureCause": null, "publishedPrefixCount": null, "publishedIpv4PrefixCount": null, "publishedIpv6PrefixCount": null, "probeSourceAddress": null, "probeSourceEvidenceClass": "DECLARED" }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:18.117Z", "limitations": [ "VANTAGE_LIMITATION: convention — Generative Metrics observes from its own network vantage with a single synthetic request sent from its own declared scan address. A request from our address can never confirm that traffic claiming a crawler name is that vendor’s crawler, because the vendor’s fleet uses its own address ranges, reverse DNS, edge signatures and script behaviour. No verification result is claimed, in either direction, by this protocol.", "PROBE_SOURCE_DECLARED: convention — Generative Metrics’ own probe source address is a DECLARED self-report supplied as an input, never a measured observation, so it can never be read as evidence about a vendor’s crawler.", "VERIFICATION_NOT_PERFORMED: convention — this protocol records the state of the published identity material. It performs no match and no mismatch between a request and an address range, and its observation result is INDETERMINATE whenever a verification cannot be made. Every state reachable from Generative Metrics’ vantage is such a case.", "Only a payload hash, a bounded prefix count and a shape verdict are recorded for each vendor document; no document body and no address value leaves the retrieval phase.", "VANTAGE_LIMITED: convention — the vendor-published identity material is in hand for at least one audited actor, and the verification itself still cannot be performed from Generative Metrics’ vantage. The material is recorded so a site owner can use it directly with their own traffic measurements." ] }, { "observationId": "61fc8799-be24-4e41-a47f-84e3e2c1a596", "protocolExecutionId": "b60ff6e7-c9dc-43d2-bfb2-eeca5d62c334", "kind": "CRAWLER_IDENTITY_VERIFICATION", "result": "NOT_OBSERVED", "evidenceClass": "DECLARED", "value": { "actorId": "google:googlebot", "actorProvider": "Google", "productToken": "Googlebot", "verificationState": "RANGES_UNAVAILABLE", "publishedRangesState": "NO_PUBLISHED_RANGES_FOR_ACTOR", "verificationPerformed": false, "vendorSystemsTookPart": false, "canVerifyVendorCrawler": false, "customerSideMeasurementInputAvailable": false, "declaredSourceId": null, "declaredSourceUrl": null, "documentRetrieved": null, "documentShape": null, "documentHttpStatus": null, "documentByteLength": null, "documentBodyTruncated": null, "documentPayloadSha256": null, "documentRetrievedAt": null, "documentFailureCause": null, "publishedPrefixCount": null, "publishedIpv4PrefixCount": null, "publishedIpv6PrefixCount": null, "probeSourceAddress": null, "probeSourceEvidenceClass": "DECLARED" }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:18.117Z", "limitations": [ "VANTAGE_LIMITATION: convention — Generative Metrics observes from its own network vantage with a single synthetic request sent from its own declared scan address. A request from our address can never confirm that traffic claiming a crawler name is that vendor’s crawler, because the vendor’s fleet uses its own address ranges, reverse DNS, edge signatures and script behaviour. No verification result is claimed, in either direction, by this protocol.", "PROBE_SOURCE_DECLARED: convention — Generative Metrics’ own probe source address is a DECLARED self-report supplied as an input, never a measured observation, so it can never be read as evidence about a vendor’s crawler.", "VERIFICATION_NOT_PERFORMED: convention — this protocol records the state of the published identity material. It performs no match and no mismatch between a request and an address range, and its observation result is INDETERMINATE whenever a verification cannot be made. Every state reachable from Generative Metrics’ vantage is such a case.", "Only a payload hash, a bounded prefix count and a shape verdict are recorded for each vendor document; no document body and no address value leaves the retrieval phase.", "VANTAGE_LIMITED: convention — the vendor-published identity material is in hand for at least one audited actor, and the verification itself still cannot be performed from Generative Metrics’ vantage. The material is recorded so a site owner can use it directly with their own traffic measurements." ] }, { "observationId": "613c2fc8-6959-469c-9a85-80a47f7a0d98", "protocolExecutionId": "b60ff6e7-c9dc-43d2-bfb2-eeca5d62c334", "kind": "CRAWLER_IDENTITY_VERIFICATION", "result": "NOT_OBSERVED", "evidenceClass": "DECLARED", "value": { "actorId": "perplexity:perplexitybot", "actorProvider": "Perplexity", "productToken": "PerplexityBot", "verificationState": "RANGES_UNAVAILABLE", "publishedRangesState": "RANGES_UNAVAILABLE", "verificationPerformed": false, "vendorSystemsTookPart": false, "canVerifyVendorCrawler": false, "customerSideMeasurementInputAvailable": false, "declaredSourceId": "perplexity:perplexitybot-json", "declaredSourceUrl": "https://www.perplexity.ai/perplexitybot.json", "documentRetrieved": null, "documentShape": null, "documentHttpStatus": null, "documentByteLength": null, "documentBodyTruncated": null, "documentPayloadSha256": null, "documentRetrievedAt": null, "documentFailureCause": null, "publishedPrefixCount": null, "publishedIpv4PrefixCount": null, "publishedIpv6PrefixCount": null, "probeSourceAddress": null, "probeSourceEvidenceClass": "DECLARED" }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:18.117Z", "limitations": [ "VANTAGE_LIMITATION: convention — Generative Metrics observes from its own network vantage with a single synthetic request sent from its own declared scan address. A request from our address can never confirm that traffic claiming a crawler name is that vendor’s crawler, because the vendor’s fleet uses its own address ranges, reverse DNS, edge signatures and script behaviour. No verification result is claimed, in either direction, by this protocol.", "PROBE_SOURCE_DECLARED: convention — Generative Metrics’ own probe source address is a DECLARED self-report supplied as an input, never a measured observation, so it can never be read as evidence about a vendor’s crawler.", "VERIFICATION_NOT_PERFORMED: convention — this protocol records the state of the published identity material. It performs no match and no mismatch between a request and an address range, and its observation result is INDETERMINATE whenever a verification cannot be made. Every state reachable from Generative Metrics’ vantage is such a case.", "Only a payload hash, a bounded prefix count and a shape verdict are recorded for each vendor document; no document body and no address value leaves the retrieval phase.", "VANTAGE_LIMITED: convention — the vendor-published identity material is in hand for at least one audited actor, and the verification itself still cannot be performed from Generative Metrics’ vantage. The material is recorded so a site owner can use it directly with their own traffic measurements." ] }, { "observationId": "5b01e7cc-6f32-4ccb-9aff-2f5962ef3b44", "protocolExecutionId": "b60ff6e7-c9dc-43d2-bfb2-eeca5d62c334", "kind": "CRAWLER_IDENTITY_VERIFICATION", "result": "NOT_OBSERVED", "evidenceClass": "DECLARED", "value": { "actorId": "openai:gptbot", "actorProvider": "OpenAI", "productToken": "GPTBot", "verificationState": "RANGES_UNAVAILABLE", "publishedRangesState": "RANGES_UNAVAILABLE", "verificationPerformed": false, "vendorSystemsTookPart": false, "canVerifyVendorCrawler": false, "customerSideMeasurementInputAvailable": false, "declaredSourceId": "openai:gptbot-json", "declaredSourceUrl": "https://openai.com/gptbot.json", "documentRetrieved": null, "documentShape": null, "documentHttpStatus": null, "documentByteLength": null, "documentBodyTruncated": null, "documentPayloadSha256": null, "documentRetrievedAt": null, "documentFailureCause": null, "publishedPrefixCount": null, "publishedIpv4PrefixCount": null, "publishedIpv6PrefixCount": null, "probeSourceAddress": null, "probeSourceEvidenceClass": "DECLARED" }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:18.117Z", "limitations": [ "VANTAGE_LIMITATION: convention — Generative Metrics observes from its own network vantage with a single synthetic request sent from its own declared scan address. A request from our address can never confirm that traffic claiming a crawler name is that vendor’s crawler, because the vendor’s fleet uses its own address ranges, reverse DNS, edge signatures and script behaviour. No verification result is claimed, in either direction, by this protocol.", "PROBE_SOURCE_DECLARED: convention — Generative Metrics’ own probe source address is a DECLARED self-report supplied as an input, never a measured observation, so it can never be read as evidence about a vendor’s crawler.", "VERIFICATION_NOT_PERFORMED: convention — this protocol records the state of the published identity material. It performs no match and no mismatch between a request and an address range, and its observation result is INDETERMINATE whenever a verification cannot be made. Every state reachable from Generative Metrics’ vantage is such a case.", "Only a payload hash, a bounded prefix count and a shape verdict are recorded for each vendor document; no document body and no address value leaves the retrieval phase.", "VANTAGE_LIMITED: convention — the vendor-published identity material is in hand for at least one audited actor, and the verification itself still cannot be performed from Generative Metrics’ vantage. The material is recorded so a site owner can use it directly with their own traffic measurements." ] }, { "observationId": "4411197a-ab52-4291-b9c0-f64d4018703b", "protocolExecutionId": "b60ff6e7-c9dc-43d2-bfb2-eeca5d62c334", "kind": "CRAWLER_IDENTITY_VERIFICATION", "result": "NOT_OBSERVED", "evidenceClass": "DECLARED", "value": { "actorId": "anthropic:claudebot", "actorProvider": "Anthropic", "productToken": "ClaudeBot", "verificationState": "RANGES_UNAVAILABLE", "publishedRangesState": "RANGES_UNAVAILABLE", "verificationPerformed": false, "vendorSystemsTookPart": false, "canVerifyVendorCrawler": false, "customerSideMeasurementInputAvailable": false, "declaredSourceId": "anthropic:bots-json", "declaredSourceUrl": "https://claude.com/crawling/bots.json", "documentRetrieved": null, "documentShape": null, "documentHttpStatus": null, "documentByteLength": null, "documentBodyTruncated": null, "documentPayloadSha256": null, "documentRetrievedAt": null, "documentFailureCause": null, "publishedPrefixCount": null, "publishedIpv4PrefixCount": null, "publishedIpv6PrefixCount": null, "probeSourceAddress": null, "probeSourceEvidenceClass": "DECLARED" }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:18.117Z", "limitations": [ "VANTAGE_LIMITATION: convention — Generative Metrics observes from its own network vantage with a single synthetic request sent from its own declared scan address. A request from our address can never confirm that traffic claiming a crawler name is that vendor’s crawler, because the vendor’s fleet uses its own address ranges, reverse DNS, edge signatures and script behaviour. No verification result is claimed, in either direction, by this protocol.", "PROBE_SOURCE_DECLARED: convention — Generative Metrics’ own probe source address is a DECLARED self-report supplied as an input, never a measured observation, so it can never be read as evidence about a vendor’s crawler.", "VERIFICATION_NOT_PERFORMED: convention — this protocol records the state of the published identity material. It performs no match and no mismatch between a request and an address range, and its observation result is INDETERMINATE whenever a verification cannot be made. Every state reachable from Generative Metrics’ vantage is such a case.", "Only a payload hash, a bounded prefix count and a shape verdict are recorded for each vendor document; no document body and no address value leaves the retrieval phase.", "VANTAGE_LIMITED: convention — the vendor-published identity material is in hand for at least one audited actor, and the verification itself still cannot be performed from Generative Metrics’ vantage. The material is recorded so a site owner can use it directly with their own traffic measurements." ] }, { "observationId": "5c1f1745-85b2-409a-8af0-36660a643f21", "protocolExecutionId": "b60ff6e7-c9dc-43d2-bfb2-eeca5d62c334", "kind": "CRAWLER_IDENTITY_VERIFICATION", "result": "NOT_OBSERVED", "evidenceClass": "DECLARED", "value": { "actorId": "google:google-extended", "actorProvider": "Google", "productToken": "Google-Extended", "verificationState": "RANGES_UNAVAILABLE", "publishedRangesState": "NO_PUBLISHED_RANGES_FOR_ACTOR", "verificationPerformed": false, "vendorSystemsTookPart": false, "canVerifyVendorCrawler": false, "customerSideMeasurementInputAvailable": false, "declaredSourceId": null, "declaredSourceUrl": null, "documentRetrieved": null, "documentShape": null, "documentHttpStatus": null, "documentByteLength": null, "documentBodyTruncated": null, "documentPayloadSha256": null, "documentRetrievedAt": null, "documentFailureCause": null, "publishedPrefixCount": null, "publishedIpv4PrefixCount": null, "publishedIpv6PrefixCount": null, "probeSourceAddress": null, "probeSourceEvidenceClass": "DECLARED", "controlTokenNote": "This actor is a declared preference token rather than an HTTP fetcher, so no request-based identity comparison is possible even in principle." }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:18.117Z", "limitations": [ "VANTAGE_LIMITATION: convention — Generative Metrics observes from its own network vantage with a single synthetic request sent from its own declared scan address. A request from our address can never confirm that traffic claiming a crawler name is that vendor’s crawler, because the vendor’s fleet uses its own address ranges, reverse DNS, edge signatures and script behaviour. No verification result is claimed, in either direction, by this protocol.", "PROBE_SOURCE_DECLARED: convention — Generative Metrics’ own probe source address is a DECLARED self-report supplied as an input, never a measured observation, so it can never be read as evidence about a vendor’s crawler.", "VERIFICATION_NOT_PERFORMED: convention — this protocol records the state of the published identity material. It performs no match and no mismatch between a request and an address range, and its observation result is INDETERMINATE whenever a verification cannot be made. Every state reachable from Generative Metrics’ vantage is such a case.", "Only a payload hash, a bounded prefix count and a shape verdict are recorded for each vendor document; no document body and no address value leaves the retrieval phase.", "VANTAGE_LIMITED: convention — the vendor-published identity material is in hand for at least one audited actor, and the verification itself still cannot be performed from Generative Metrics’ vantage. The material is recorded so a site owner can use it directly with their own traffic measurements." ] }, { "observationId": "3ae4ed25-d971-4994-ac47-877eded7706e", "protocolExecutionId": "b60ff6e7-c9dc-43d2-bfb2-eeca5d62c334", "kind": "CRAWLER_IDENTITY_VERIFICATION", "result": "NOT_OBSERVED", "evidenceClass": "DECLARED", "value": { "actorId": "openai:chatgpt-user", "actorProvider": "OpenAI", "productToken": "ChatGPT-User", "verificationState": "RANGES_UNAVAILABLE", "publishedRangesState": "RANGES_UNAVAILABLE", "verificationPerformed": false, "vendorSystemsTookPart": false, "canVerifyVendorCrawler": false, "customerSideMeasurementInputAvailable": false, "declaredSourceId": "openai:chatgpt-user-json", "declaredSourceUrl": "https://openai.com/chatgpt-user.json", "documentRetrieved": null, "documentShape": null, "documentHttpStatus": null, "documentByteLength": null, "documentBodyTruncated": null, "documentPayloadSha256": null, "documentRetrievedAt": null, "documentFailureCause": null, "publishedPrefixCount": null, "publishedIpv4PrefixCount": null, "publishedIpv6PrefixCount": null, "probeSourceAddress": null, "probeSourceEvidenceClass": "DECLARED" }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:18.117Z", "limitations": [ "VANTAGE_LIMITATION: convention — Generative Metrics observes from its own network vantage with a single synthetic request sent from its own declared scan address. A request from our address can never confirm that traffic claiming a crawler name is that vendor’s crawler, because the vendor’s fleet uses its own address ranges, reverse DNS, edge signatures and script behaviour. No verification result is claimed, in either direction, by this protocol.", "PROBE_SOURCE_DECLARED: convention — Generative Metrics’ own probe source address is a DECLARED self-report supplied as an input, never a measured observation, so it can never be read as evidence about a vendor’s crawler.", "VERIFICATION_NOT_PERFORMED: convention — this protocol records the state of the published identity material. It performs no match and no mismatch between a request and an address range, and its observation result is INDETERMINATE whenever a verification cannot be made. Every state reachable from Generative Metrics’ vantage is such a case.", "Only a payload hash, a bounded prefix count and a shape verdict are recorded for each vendor document; no document body and no address value leaves the retrieval phase.", "VANTAGE_LIMITED: convention — the vendor-published identity material is in hand for at least one audited actor, and the verification itself still cannot be performed from Generative Metrics’ vantage. The material is recorded so a site owner can use it directly with their own traffic measurements." ] }, { "observationId": "9fb467c2-f85c-4d24-a7d6-d78335a8285a", "protocolExecutionId": "b60ff6e7-c9dc-43d2-bfb2-eeca5d62c334", "kind": "CRAWLER_IDENTITY_VERIFICATION", "result": "NOT_OBSERVED", "evidenceClass": "DECLARED", "value": { "actorId": "anthropic:claude-user", "actorProvider": "Anthropic", "productToken": "Claude-User", "verificationState": "RANGES_UNAVAILABLE", "publishedRangesState": "RANGES_UNAVAILABLE", "verificationPerformed": false, "vendorSystemsTookPart": false, "canVerifyVendorCrawler": false, "customerSideMeasurementInputAvailable": false, "declaredSourceId": "anthropic:bots-json", "declaredSourceUrl": "https://claude.com/crawling/bots.json", "documentRetrieved": null, "documentShape": null, "documentHttpStatus": null, "documentByteLength": null, "documentBodyTruncated": null, "documentPayloadSha256": null, "documentRetrievedAt": null, "documentFailureCause": null, "publishedPrefixCount": null, "publishedIpv4PrefixCount": null, "publishedIpv6PrefixCount": null, "probeSourceAddress": null, "probeSourceEvidenceClass": "DECLARED" }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:18.117Z", "limitations": [ "VANTAGE_LIMITATION: convention — Generative Metrics observes from its own network vantage with a single synthetic request sent from its own declared scan address. A request from our address can never confirm that traffic claiming a crawler name is that vendor’s crawler, because the vendor’s fleet uses its own address ranges, reverse DNS, edge signatures and script behaviour. No verification result is claimed, in either direction, by this protocol.", "PROBE_SOURCE_DECLARED: convention — Generative Metrics’ own probe source address is a DECLARED self-report supplied as an input, never a measured observation, so it can never be read as evidence about a vendor’s crawler.", "VERIFICATION_NOT_PERFORMED: convention — this protocol records the state of the published identity material. It performs no match and no mismatch between a request and an address range, and its observation result is INDETERMINATE whenever a verification cannot be made. Every state reachable from Generative Metrics’ vantage is such a case.", "Only a payload hash, a bounded prefix count and a shape verdict are recorded for each vendor document; no document body and no address value leaves the retrieval phase.", "VANTAGE_LIMITED: convention — the vendor-published identity material is in hand for at least one audited actor, and the verification itself still cannot be performed from Generative Metrics’ vantage. The material is recorded so a site owner can use it directly with their own traffic measurements." ] }, { "observationId": "8e904e6c-4cc6-4f01-bd2e-b057279c0bd4", "protocolExecutionId": "b60ff6e7-c9dc-43d2-bfb2-eeca5d62c334", "kind": "CRAWLER_IDENTITY_VERIFICATION", "result": "NOT_OBSERVED", "evidenceClass": "DECLARED", "value": { "actorId": "perplexity:perplexity-user", "actorProvider": "Perplexity", "productToken": "Perplexity-User", "verificationState": "RANGES_UNAVAILABLE", "publishedRangesState": "NO_PUBLISHED_RANGES_FOR_ACTOR", "verificationPerformed": false, "vendorSystemsTookPart": false, "canVerifyVendorCrawler": false, "customerSideMeasurementInputAvailable": false, "declaredSourceId": null, "declaredSourceUrl": null, "documentRetrieved": null, "documentShape": null, "documentHttpStatus": null, "documentByteLength": null, "documentBodyTruncated": null, "documentPayloadSha256": null, "documentRetrievedAt": null, "documentFailureCause": null, "publishedPrefixCount": null, "publishedIpv4PrefixCount": null, "publishedIpv6PrefixCount": null, "probeSourceAddress": null, "probeSourceEvidenceClass": "DECLARED" }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:18.117Z", "limitations": [ "VANTAGE_LIMITATION: convention — Generative Metrics observes from its own network vantage with a single synthetic request sent from its own declared scan address. A request from our address can never confirm that traffic claiming a crawler name is that vendor’s crawler, because the vendor’s fleet uses its own address ranges, reverse DNS, edge signatures and script behaviour. No verification result is claimed, in either direction, by this protocol.", "PROBE_SOURCE_DECLARED: convention — Generative Metrics’ own probe source address is a DECLARED self-report supplied as an input, never a measured observation, so it can never be read as evidence about a vendor’s crawler.", "VERIFICATION_NOT_PERFORMED: convention — this protocol records the state of the published identity material. It performs no match and no mismatch between a request and an address range, and its observation result is INDETERMINATE whenever a verification cannot be made. Every state reachable from Generative Metrics’ vantage is such a case.", "Only a payload hash, a bounded prefix count and a shape verdict are recorded for each vendor document; no document body and no address value leaves the retrieval phase.", "VANTAGE_LIMITED: convention — the vendor-published identity material is in hand for at least one audited actor, and the verification itself still cannot be performed from Generative Metrics’ vantage. The material is recorded so a site owner can use it directly with their own traffic measurements." ] }, { "observationId": "8f6c72d1-a2de-47a3-893f-286347be9a82", "protocolExecutionId": "b60ff6e7-c9dc-43d2-bfb2-eeca5d62c334", "kind": "CRAWLER_IDENTITY_VERIFICATION", "result": "NOT_OBSERVED", "evidenceClass": "DECLARED", "value": { "actorId": "google:google-agent", "actorProvider": "Google", "productToken": "Google-Agent", "verificationState": "RANGES_UNAVAILABLE", "publishedRangesState": "NO_PUBLISHED_RANGES_FOR_ACTOR", "verificationPerformed": false, "vendorSystemsTookPart": false, "canVerifyVendorCrawler": false, "customerSideMeasurementInputAvailable": false, "declaredSourceId": null, "declaredSourceUrl": null, "documentRetrieved": null, "documentShape": null, "documentHttpStatus": null, "documentByteLength": null, "documentBodyTruncated": null, "documentPayloadSha256": null, "documentRetrievedAt": null, "documentFailureCause": null, "publishedPrefixCount": null, "publishedIpv4PrefixCount": null, "publishedIpv6PrefixCount": null, "probeSourceAddress": null, "probeSourceEvidenceClass": "DECLARED" }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:18.117Z", "limitations": [ "VANTAGE_LIMITATION: convention — Generative Metrics observes from its own network vantage with a single synthetic request sent from its own declared scan address. A request from our address can never confirm that traffic claiming a crawler name is that vendor’s crawler, because the vendor’s fleet uses its own address ranges, reverse DNS, edge signatures and script behaviour. No verification result is claimed, in either direction, by this protocol.", "PROBE_SOURCE_DECLARED: convention — Generative Metrics’ own probe source address is a DECLARED self-report supplied as an input, never a measured observation, so it can never be read as evidence about a vendor’s crawler.", "VERIFICATION_NOT_PERFORMED: convention — this protocol records the state of the published identity material. It performs no match and no mismatch between a request and an address range, and its observation result is INDETERMINATE whenever a verification cannot be made. Every state reachable from Generative Metrics’ vantage is such a case.", "Only a payload hash, a bounded prefix count and a shape verdict are recorded for each vendor document; no document body and no address value leaves the retrieval phase.", "VANTAGE_LIMITED: convention — the vendor-published identity material is in hand for at least one audited actor, and the verification itself still cannot be performed from Generative Metrics’ vantage. The material is recorded so a site owner can use it directly with their own traffic measurements." ] }, { "observationId": "8cd10223-f172-4eea-90a5-b900efe829cd", "protocolExecutionId": "b60ff6e7-c9dc-43d2-bfb2-eeca5d62c334", "kind": "CRAWLER_IDENTITY_VERIFICATION", "result": "NOT_OBSERVED", "evidenceClass": "DECLARED", "value": { "actorId": "openai:oai-adsbot", "actorProvider": "OpenAI", "productToken": "OAI-AdsBot", "verificationState": "RANGES_UNAVAILABLE", "publishedRangesState": "RANGES_UNAVAILABLE", "verificationPerformed": false, "vendorSystemsTookPart": false, "canVerifyVendorCrawler": false, "customerSideMeasurementInputAvailable": false, "declaredSourceId": "openai:adsbot-json", "declaredSourceUrl": "https://openai.com/adsbot.json", "documentRetrieved": null, "documentShape": null, "documentHttpStatus": null, "documentByteLength": null, "documentBodyTruncated": null, "documentPayloadSha256": null, "documentRetrievedAt": null, "documentFailureCause": null, "publishedPrefixCount": null, "publishedIpv4PrefixCount": null, "publishedIpv6PrefixCount": null, "probeSourceAddress": null, "probeSourceEvidenceClass": "DECLARED" }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:18.117Z", "limitations": [ "VANTAGE_LIMITATION: convention — Generative Metrics observes from its own network vantage with a single synthetic request sent from its own declared scan address. A request from our address can never confirm that traffic claiming a crawler name is that vendor’s crawler, because the vendor’s fleet uses its own address ranges, reverse DNS, edge signatures and script behaviour. No verification result is claimed, in either direction, by this protocol.", "PROBE_SOURCE_DECLARED: convention — Generative Metrics’ own probe source address is a DECLARED self-report supplied as an input, never a measured observation, so it can never be read as evidence about a vendor’s crawler.", "VERIFICATION_NOT_PERFORMED: convention — this protocol records the state of the published identity material. It performs no match and no mismatch between a request and an address range, and its observation result is INDETERMINATE whenever a verification cannot be made. Every state reachable from Generative Metrics’ vantage is such a case.", "Only a payload hash, a bounded prefix count and a shape verdict are recorded for each vendor document; no document body and no address value leaves the retrieval phase.", "VANTAGE_LIMITED: convention — the vendor-published identity material is in hand for at least one audited actor, and the verification itself still cannot be performed from Generative Metrics’ vantage. The material is recorded so a site owner can use it directly with their own traffic measurements." ] }, { "observationId": "d96a7587-141c-4ba5-9d5f-aa7dce902810", "protocolExecutionId": "b60ff6e7-c9dc-43d2-bfb2-eeca5d62c334", "kind": "CRAWLER_IDENTITY_VERIFICATION", "result": "NOT_OBSERVED", "evidenceClass": "DECLARED", "value": { "actorId": "kagi:kagibot", "actorProvider": "Kagi", "productToken": "Kagibot", "verificationState": "RANGES_UNAVAILABLE", "publishedRangesState": "NO_PUBLISHED_RANGES_FOR_ACTOR", "verificationPerformed": false, "vendorSystemsTookPart": false, "canVerifyVendorCrawler": false, "customerSideMeasurementInputAvailable": false, "declaredSourceId": null, "declaredSourceUrl": null, "documentRetrieved": null, "documentShape": null, "documentHttpStatus": null, "documentByteLength": null, "documentBodyTruncated": null, "documentPayloadSha256": null, "documentRetrievedAt": null, "documentFailureCause": null, "publishedPrefixCount": null, "publishedIpv4PrefixCount": null, "publishedIpv6PrefixCount": null, "probeSourceAddress": null, "probeSourceEvidenceClass": "DECLARED" }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:18.117Z", "limitations": [ "VANTAGE_LIMITATION: convention — Generative Metrics observes from its own network vantage with a single synthetic request sent from its own declared scan address. A request from our address can never confirm that traffic claiming a crawler name is that vendor’s crawler, because the vendor’s fleet uses its own address ranges, reverse DNS, edge signatures and script behaviour. No verification result is claimed, in either direction, by this protocol.", "PROBE_SOURCE_DECLARED: convention — Generative Metrics’ own probe source address is a DECLARED self-report supplied as an input, never a measured observation, so it can never be read as evidence about a vendor’s crawler.", "VERIFICATION_NOT_PERFORMED: convention — this protocol records the state of the published identity material. It performs no match and no mismatch between a request and an address range, and its observation result is INDETERMINATE whenever a verification cannot be made. Every state reachable from Generative Metrics’ vantage is such a case.", "Only a payload hash, a bounded prefix count and a shape verdict are recorded for each vendor document; no document body and no address value leaves the retrieval phase.", "VANTAGE_LIMITED: convention — the vendor-published identity material is in hand for at least one audited actor, and the verification itself still cannot be performed from Generative Metrics’ vantage. The material is recorded so a site owner can use it directly with their own traffic measurements." ] }, { "observationId": "3d425418-1bde-4cd5-90c3-49b195a9b360", "protocolExecutionId": "b60ff6e7-c9dc-43d2-bfb2-eeca5d62c334", "kind": "CRAWLER_IDENTITY_VERIFICATION", "result": "NOT_OBSERVED", "evidenceClass": "DECLARED", "value": { "actorId": "commoncrawl:ccbot", "actorProvider": "Common Crawl", "productToken": "CCBot", "verificationState": "RANGES_UNAVAILABLE", "publishedRangesState": "NO_PUBLISHED_RANGES_FOR_ACTOR", "verificationPerformed": false, "vendorSystemsTookPart": false, "canVerifyVendorCrawler": false, "customerSideMeasurementInputAvailable": false, "declaredSourceId": null, "declaredSourceUrl": null, "documentRetrieved": null, "documentShape": null, "documentHttpStatus": null, "documentByteLength": null, "documentBodyTruncated": null, "documentPayloadSha256": null, "documentRetrievedAt": null, "documentFailureCause": null, "publishedPrefixCount": null, "publishedIpv4PrefixCount": null, "publishedIpv6PrefixCount": null, "probeSourceAddress": null, "probeSourceEvidenceClass": "DECLARED" }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:18.117Z", "limitations": [ "VANTAGE_LIMITATION: convention — Generative Metrics observes from its own network vantage with a single synthetic request sent from its own declared scan address. A request from our address can never confirm that traffic claiming a crawler name is that vendor’s crawler, because the vendor’s fleet uses its own address ranges, reverse DNS, edge signatures and script behaviour. No verification result is claimed, in either direction, by this protocol.", "PROBE_SOURCE_DECLARED: convention — Generative Metrics’ own probe source address is a DECLARED self-report supplied as an input, never a measured observation, so it can never be read as evidence about a vendor’s crawler.", "VERIFICATION_NOT_PERFORMED: convention — this protocol records the state of the published identity material. It performs no match and no mismatch between a request and an address range, and its observation result is INDETERMINATE whenever a verification cannot be made. Every state reachable from Generative Metrics’ vantage is such a case.", "Only a payload hash, a bounded prefix count and a shape verdict are recorded for each vendor document; no document body and no address value leaves the retrieval phase.", "VANTAGE_LIMITED: convention — the vendor-published identity material is in hand for at least one audited actor, and the verification itself still cannot be performed from Generative Metrics’ vantage. The material is recorded so a site owner can use it directly with their own traffic measurements." ] }, { "observationId": "a980364a-90c0-4ef6-b870-0655425cc85f", "protocolExecutionId": "b60ff6e7-c9dc-43d2-bfb2-eeca5d62c334", "kind": "CRAWLER_IDENTITY_VERIFICATION", "result": "NOT_OBSERVED", "evidenceClass": "DECLARED", "value": { "actorId": "microsoft:bingbot", "actorProvider": "Microsoft", "productToken": "bingbot", "verificationState": "RANGES_UNAVAILABLE", "publishedRangesState": "NO_PUBLISHED_RANGES_FOR_ACTOR", "verificationPerformed": false, "vendorSystemsTookPart": false, "canVerifyVendorCrawler": false, "customerSideMeasurementInputAvailable": false, "declaredSourceId": null, "declaredSourceUrl": null, "documentRetrieved": null, "documentShape": null, "documentHttpStatus": null, "documentByteLength": null, "documentBodyTruncated": null, "documentPayloadSha256": null, "documentRetrievedAt": null, "documentFailureCause": null, "publishedPrefixCount": null, "publishedIpv4PrefixCount": null, "publishedIpv6PrefixCount": null, "probeSourceAddress": null, "probeSourceEvidenceClass": "DECLARED" }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:18.117Z", "limitations": [ "VANTAGE_LIMITATION: convention — Generative Metrics observes from its own network vantage with a single synthetic request sent from its own declared scan address. A request from our address can never confirm that traffic claiming a crawler name is that vendor’s crawler, because the vendor’s fleet uses its own address ranges, reverse DNS, edge signatures and script behaviour. No verification result is claimed, in either direction, by this protocol.", "PROBE_SOURCE_DECLARED: convention — Generative Metrics’ own probe source address is a DECLARED self-report supplied as an input, never a measured observation, so it can never be read as evidence about a vendor’s crawler.", "VERIFICATION_NOT_PERFORMED: convention — this protocol records the state of the published identity material. It performs no match and no mismatch between a request and an address range, and its observation result is INDETERMINATE whenever a verification cannot be made. Every state reachable from Generative Metrics’ vantage is such a case.", "Only a payload hash, a bounded prefix count and a shape verdict are recorded for each vendor document; no document body and no address value leaves the retrieval phase.", "VANTAGE_LIMITED: convention — the vendor-published identity material is in hand for at least one audited actor, and the verification itself still cannot be performed from Generative Metrics’ vantage. The material is recorded so a site owner can use it directly with their own traffic measurements." ] }, { "observationId": "bbfe07cf-a5ef-40c8-bf1f-53a700a6dad0", "protocolExecutionId": "b60ff6e7-c9dc-43d2-bfb2-eeca5d62c334", "kind": "CRAWLER_IDENTITY_VERIFICATION", "result": "NOT_OBSERVED", "evidenceClass": "DECLARED", "value": { "actorId": "apple:applebot", "actorProvider": "Apple", "productToken": "Applebot", "verificationState": "RANGES_UNAVAILABLE", "publishedRangesState": "NO_PUBLISHED_RANGES_FOR_ACTOR", "verificationPerformed": false, "vendorSystemsTookPart": false, "canVerifyVendorCrawler": false, "customerSideMeasurementInputAvailable": false, "declaredSourceId": null, "declaredSourceUrl": null, "documentRetrieved": null, "documentShape": null, "documentHttpStatus": null, "documentByteLength": null, "documentBodyTruncated": null, "documentPayloadSha256": null, "documentRetrievedAt": null, "documentFailureCause": null, "publishedPrefixCount": null, "publishedIpv4PrefixCount": null, "publishedIpv6PrefixCount": null, "probeSourceAddress": null, "probeSourceEvidenceClass": "DECLARED" }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:18.117Z", "limitations": [ "VANTAGE_LIMITATION: convention — Generative Metrics observes from its own network vantage with a single synthetic request sent from its own declared scan address. A request from our address can never confirm that traffic claiming a crawler name is that vendor’s crawler, because the vendor’s fleet uses its own address ranges, reverse DNS, edge signatures and script behaviour. No verification result is claimed, in either direction, by this protocol.", "PROBE_SOURCE_DECLARED: convention — Generative Metrics’ own probe source address is a DECLARED self-report supplied as an input, never a measured observation, so it can never be read as evidence about a vendor’s crawler.", "VERIFICATION_NOT_PERFORMED: convention — this protocol records the state of the published identity material. It performs no match and no mismatch between a request and an address range, and its observation result is INDETERMINATE whenever a verification cannot be made. Every state reachable from Generative Metrics’ vantage is such a case.", "Only a payload hash, a bounded prefix count and a shape verdict are recorded for each vendor document; no document body and no address value leaves the retrieval phase.", "VANTAGE_LIMITED: convention — the vendor-published identity material is in hand for at least one audited actor, and the verification itself still cannot be performed from Generative Metrics’ vantage. The material is recorded so a site owner can use it directly with their own traffic measurements." ] }, { "observationId": "6f1e5ec6-f777-4a69-bff6-45f93c94c657", "protocolExecutionId": "b60ff6e7-c9dc-43d2-bfb2-eeca5d62c334", "kind": "CRAWLER_IDENTITY_VERIFICATION", "result": "NOT_OBSERVED", "evidenceClass": "DECLARED", "value": { "actorId": "apple:applebot-extended", "actorProvider": "Apple", "productToken": "Applebot-Extended", "verificationState": "RANGES_UNAVAILABLE", "publishedRangesState": "NO_PUBLISHED_RANGES_FOR_ACTOR", "verificationPerformed": false, "vendorSystemsTookPart": false, "canVerifyVendorCrawler": false, "customerSideMeasurementInputAvailable": false, "declaredSourceId": null, "declaredSourceUrl": null, "documentRetrieved": null, "documentShape": null, "documentHttpStatus": null, "documentByteLength": null, "documentBodyTruncated": null, "documentPayloadSha256": null, "documentRetrievedAt": null, "documentFailureCause": null, "publishedPrefixCount": null, "publishedIpv4PrefixCount": null, "publishedIpv6PrefixCount": null, "probeSourceAddress": null, "probeSourceEvidenceClass": "DECLARED", "controlTokenNote": "This actor is a declared preference token rather than an HTTP fetcher, so no request-based identity comparison is possible even in principle." }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:18.117Z", "limitations": [ "VANTAGE_LIMITATION: convention — Generative Metrics observes from its own network vantage with a single synthetic request sent from its own declared scan address. A request from our address can never confirm that traffic claiming a crawler name is that vendor’s crawler, because the vendor’s fleet uses its own address ranges, reverse DNS, edge signatures and script behaviour. No verification result is claimed, in either direction, by this protocol.", "PROBE_SOURCE_DECLARED: convention — Generative Metrics’ own probe source address is a DECLARED self-report supplied as an input, never a measured observation, so it can never be read as evidence about a vendor’s crawler.", "VERIFICATION_NOT_PERFORMED: convention — this protocol records the state of the published identity material. It performs no match and no mismatch between a request and an address range, and its observation result is INDETERMINATE whenever a verification cannot be made. Every state reachable from Generative Metrics’ vantage is such a case.", "Only a payload hash, a bounded prefix count and a shape verdict are recorded for each vendor document; no document body and no address value leaves the retrieval phase.", "VANTAGE_LIMITED: convention — the vendor-published identity material is in hand for at least one audited actor, and the verification itself still cannot be performed from Generative Metrics’ vantage. The material is recorded so a site owner can use it directly with their own traffic measurements." ] }, { "observationId": "34b7f0ac-e50f-4941-b117-5430c2ce4a03", "protocolExecutionId": "b60ff6e7-c9dc-43d2-bfb2-eeca5d62c334", "kind": "CRAWLER_IDENTITY_VERIFICATION", "result": "NOT_OBSERVED", "evidenceClass": "DECLARED", "value": { "actorId": "duckduckgo:duckassistbot", "actorProvider": "DuckDuckGo", "productToken": "DuckAssistBot", "verificationState": "RANGES_UNAVAILABLE", "publishedRangesState": "NO_PUBLISHED_RANGES_FOR_ACTOR", "verificationPerformed": false, "vendorSystemsTookPart": false, "canVerifyVendorCrawler": false, "customerSideMeasurementInputAvailable": false, "declaredSourceId": null, "declaredSourceUrl": null, "documentRetrieved": null, "documentShape": null, "documentHttpStatus": null, "documentByteLength": null, "documentBodyTruncated": null, "documentPayloadSha256": null, "documentRetrievedAt": null, "documentFailureCause": null, "publishedPrefixCount": null, "publishedIpv4PrefixCount": null, "publishedIpv6PrefixCount": null, "probeSourceAddress": null, "probeSourceEvidenceClass": "DECLARED" }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:18.117Z", "limitations": [ "VANTAGE_LIMITATION: convention — Generative Metrics observes from its own network vantage with a single synthetic request sent from its own declared scan address. A request from our address can never confirm that traffic claiming a crawler name is that vendor’s crawler, because the vendor’s fleet uses its own address ranges, reverse DNS, edge signatures and script behaviour. No verification result is claimed, in either direction, by this protocol.", "PROBE_SOURCE_DECLARED: convention — Generative Metrics’ own probe source address is a DECLARED self-report supplied as an input, never a measured observation, so it can never be read as evidence about a vendor’s crawler.", "VERIFICATION_NOT_PERFORMED: convention — this protocol records the state of the published identity material. It performs no match and no mismatch between a request and an address range, and its observation result is INDETERMINATE whenever a verification cannot be made. Every state reachable from Generative Metrics’ vantage is such a case.", "Only a payload hash, a bounded prefix count and a shape verdict are recorded for each vendor document; no document body and no address value leaves the retrieval phase.", "VANTAGE_LIMITED: convention — the vendor-published identity material is in hand for at least one audited actor, and the verification itself still cannot be performed from Generative Metrics’ vantage. The material is recorded so a site owner can use it directly with their own traffic measurements." ] }, { "observationId": "f5c8c18e-ac0e-4b50-9a4f-63957ee157c4", "protocolExecutionId": "b60ff6e7-c9dc-43d2-bfb2-eeca5d62c334", "kind": "CRAWLER_IDENTITY_VERIFICATION", "result": "NOT_OBSERVED", "evidenceClass": "DECLARED", "value": { "actorId": "diffbot:diffbot", "actorProvider": "Diffbot", "productToken": "Diffbot", "verificationState": "RANGES_UNAVAILABLE", "publishedRangesState": "NO_PUBLISHED_RANGES_FOR_ACTOR", "verificationPerformed": false, "vendorSystemsTookPart": false, "canVerifyVendorCrawler": false, "customerSideMeasurementInputAvailable": false, "declaredSourceId": null, "declaredSourceUrl": null, "documentRetrieved": null, "documentShape": null, "documentHttpStatus": null, "documentByteLength": null, "documentBodyTruncated": null, "documentPayloadSha256": null, "documentRetrievedAt": null, "documentFailureCause": null, "publishedPrefixCount": null, "publishedIpv4PrefixCount": null, "publishedIpv6PrefixCount": null, "probeSourceAddress": null, "probeSourceEvidenceClass": "DECLARED" }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:18.117Z", "limitations": [ "VANTAGE_LIMITATION: convention — Generative Metrics observes from its own network vantage with a single synthetic request sent from its own declared scan address. A request from our address can never confirm that traffic claiming a crawler name is that vendor’s crawler, because the vendor’s fleet uses its own address ranges, reverse DNS, edge signatures and script behaviour. No verification result is claimed, in either direction, by this protocol.", "PROBE_SOURCE_DECLARED: convention — Generative Metrics’ own probe source address is a DECLARED self-report supplied as an input, never a measured observation, so it can never be read as evidence about a vendor’s crawler.", "VERIFICATION_NOT_PERFORMED: convention — this protocol records the state of the published identity material. It performs no match and no mismatch between a request and an address range, and its observation result is INDETERMINATE whenever a verification cannot be made. Every state reachable from Generative Metrics’ vantage is such a case.", "Only a payload hash, a bounded prefix count and a shape verdict are recorded for each vendor document; no document body and no address value leaves the retrieval phase.", "VANTAGE_LIMITED: convention — the vendor-published identity material is in hand for at least one audited actor, and the verification itself still cannot be performed from Generative Metrics’ vantage. The material is recorded so a site owner can use it directly with their own traffic measurements." ] }, { "observationId": "39fc53e9-dc7e-4119-81d0-dccfcb929db3", "protocolExecutionId": "b60ff6e7-c9dc-43d2-bfb2-eeca5d62c334", "kind": "CRAWLER_IDENTITY_VERIFICATION", "result": "NOT_OBSERVED", "evidenceClass": "DECLARED", "value": { "actorId": "webzio:omgili", "actorProvider": "Webz.io", "productToken": "omgili", "verificationState": "RANGES_UNAVAILABLE", "publishedRangesState": "NO_PUBLISHED_RANGES_FOR_ACTOR", "verificationPerformed": false, "vendorSystemsTookPart": false, "canVerifyVendorCrawler": false, "customerSideMeasurementInputAvailable": false, "declaredSourceId": null, "declaredSourceUrl": null, "documentRetrieved": null, "documentShape": null, "documentHttpStatus": null, "documentByteLength": null, "documentBodyTruncated": null, "documentPayloadSha256": null, "documentRetrievedAt": null, "documentFailureCause": null, "publishedPrefixCount": null, "publishedIpv4PrefixCount": null, "publishedIpv6PrefixCount": null, "probeSourceAddress": null, "probeSourceEvidenceClass": "DECLARED" }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:18.117Z", "limitations": [ "VANTAGE_LIMITATION: convention — Generative Metrics observes from its own network vantage with a single synthetic request sent from its own declared scan address. A request from our address can never confirm that traffic claiming a crawler name is that vendor’s crawler, because the vendor’s fleet uses its own address ranges, reverse DNS, edge signatures and script behaviour. No verification result is claimed, in either direction, by this protocol.", "PROBE_SOURCE_DECLARED: convention — Generative Metrics’ own probe source address is a DECLARED self-report supplied as an input, never a measured observation, so it can never be read as evidence about a vendor’s crawler.", "VERIFICATION_NOT_PERFORMED: convention — this protocol records the state of the published identity material. It performs no match and no mismatch between a request and an address range, and its observation result is INDETERMINATE whenever a verification cannot be made. Every state reachable from Generative Metrics’ vantage is such a case.", "Only a payload hash, a bounded prefix count and a shape verdict are recorded for each vendor document; no document body and no address value leaves the retrieval phase.", "VANTAGE_LIMITED: convention — the vendor-published identity material is in hand for at least one audited actor, and the verification itself still cannot be performed from Generative Metrics’ vantage. The material is recorded so a site owner can use it directly with their own traffic measurements." ] }, { "observationId": "eecac4b0-a2a5-4f04-be3b-693d04a321f3", "protocolExecutionId": "b60ff6e7-c9dc-43d2-bfb2-eeca5d62c334", "kind": "CRAWLER_IDENTITY_VERIFICATION", "result": "NOT_OBSERVED", "evidenceClass": "DECLARED", "value": { "actorId": "allenai:ai2bot", "actorProvider": "Allen Institute for AI", "productToken": "AI2Bot", "verificationState": "RANGES_UNAVAILABLE", "publishedRangesState": "NO_PUBLISHED_RANGES_FOR_ACTOR", "verificationPerformed": false, "vendorSystemsTookPart": false, "canVerifyVendorCrawler": false, "customerSideMeasurementInputAvailable": false, "declaredSourceId": null, "declaredSourceUrl": null, "documentRetrieved": null, "documentShape": null, "documentHttpStatus": null, "documentByteLength": null, "documentBodyTruncated": null, "documentPayloadSha256": null, "documentRetrievedAt": null, "documentFailureCause": null, "publishedPrefixCount": null, "publishedIpv4PrefixCount": null, "publishedIpv6PrefixCount": null, "probeSourceAddress": null, "probeSourceEvidenceClass": "DECLARED" }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:18.117Z", "limitations": [ "VANTAGE_LIMITATION: convention — Generative Metrics observes from its own network vantage with a single synthetic request sent from its own declared scan address. A request from our address can never confirm that traffic claiming a crawler name is that vendor’s crawler, because the vendor’s fleet uses its own address ranges, reverse DNS, edge signatures and script behaviour. No verification result is claimed, in either direction, by this protocol.", "PROBE_SOURCE_DECLARED: convention — Generative Metrics’ own probe source address is a DECLARED self-report supplied as an input, never a measured observation, so it can never be read as evidence about a vendor’s crawler.", "VERIFICATION_NOT_PERFORMED: convention — this protocol records the state of the published identity material. It performs no match and no mismatch between a request and an address range, and its observation result is INDETERMINATE whenever a verification cannot be made. Every state reachable from Generative Metrics’ vantage is such a case.", "Only a payload hash, a bounded prefix count and a shape verdict are recorded for each vendor document; no document body and no address value leaves the retrieval phase.", "VANTAGE_LIMITED: convention — the vendor-published identity material is in hand for at least one audited actor, and the verification itself still cannot be performed from Generative Metrics’ vantage. The material is recorded so a site owner can use it directly with their own traffic measurements." ] }, { "observationId": "8844f1d6-523f-40fe-85aa-88fb409828fb", "protocolExecutionId": "b60ff6e7-c9dc-43d2-bfb2-eeca5d62c334", "kind": "CRAWLER_IDENTITY_VERIFICATION", "result": "NOT_OBSERVED", "evidenceClass": "DECLARED", "value": { "actorId": "hive:imagesiftbot", "actorProvider": "Hive", "productToken": "ImagesiftBot", "verificationState": "RANGES_UNAVAILABLE", "publishedRangesState": "NO_PUBLISHED_RANGES_FOR_ACTOR", "verificationPerformed": false, "vendorSystemsTookPart": false, "canVerifyVendorCrawler": false, "customerSideMeasurementInputAvailable": false, "declaredSourceId": null, "declaredSourceUrl": null, "documentRetrieved": null, "documentShape": null, "documentHttpStatus": null, "documentByteLength": null, "documentBodyTruncated": null, "documentPayloadSha256": null, "documentRetrievedAt": null, "documentFailureCause": null, "publishedPrefixCount": null, "publishedIpv4PrefixCount": null, "publishedIpv6PrefixCount": null, "probeSourceAddress": null, "probeSourceEvidenceClass": "DECLARED" }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:18.117Z", "limitations": [ "VANTAGE_LIMITATION: convention — Generative Metrics observes from its own network vantage with a single synthetic request sent from its own declared scan address. A request from our address can never confirm that traffic claiming a crawler name is that vendor’s crawler, because the vendor’s fleet uses its own address ranges, reverse DNS, edge signatures and script behaviour. No verification result is claimed, in either direction, by this protocol.", "PROBE_SOURCE_DECLARED: convention — Generative Metrics’ own probe source address is a DECLARED self-report supplied as an input, never a measured observation, so it can never be read as evidence about a vendor’s crawler.", "VERIFICATION_NOT_PERFORMED: convention — this protocol records the state of the published identity material. It performs no match and no mismatch between a request and an address range, and its observation result is INDETERMINATE whenever a verification cannot be made. Every state reachable from Generative Metrics’ vantage is such a case.", "Only a payload hash, a bounded prefix count and a shape verdict are recorded for each vendor document; no document body and no address value leaves the retrieval phase.", "VANTAGE_LIMITED: convention — the vendor-published identity material is in hand for at least one audited actor, and the verification itself still cannot be performed from Generative Metrics’ vantage. The material is recorded so a site owner can use it directly with their own traffic measurements." ] }, { "observationId": "fedb72c2-1cf6-4bae-886c-f712934f2e93", "protocolExecutionId": "b60ff6e7-c9dc-43d2-bfb2-eeca5d62c334", "kind": "CRAWLER_IDENTITY_VERIFICATION", "result": "NOT_OBSERVED", "evidenceClass": "DECLARED", "value": { "actorId": "linkup:linkupbot", "actorProvider": "Linkup", "productToken": "LinkupBot", "verificationState": "RANGES_UNAVAILABLE", "publishedRangesState": "NO_PUBLISHED_RANGES_FOR_ACTOR", "verificationPerformed": false, "vendorSystemsTookPart": false, "canVerifyVendorCrawler": false, "customerSideMeasurementInputAvailable": false, "declaredSourceId": null, "declaredSourceUrl": null, "documentRetrieved": null, "documentShape": null, "documentHttpStatus": null, "documentByteLength": null, "documentBodyTruncated": null, "documentPayloadSha256": null, "documentRetrievedAt": null, "documentFailureCause": null, "publishedPrefixCount": null, "publishedIpv4PrefixCount": null, "publishedIpv6PrefixCount": null, "probeSourceAddress": null, "probeSourceEvidenceClass": "DECLARED" }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:18.117Z", "limitations": [ "VANTAGE_LIMITATION: convention — Generative Metrics observes from its own network vantage with a single synthetic request sent from its own declared scan address. A request from our address can never confirm that traffic claiming a crawler name is that vendor’s crawler, because the vendor’s fleet uses its own address ranges, reverse DNS, edge signatures and script behaviour. No verification result is claimed, in either direction, by this protocol.", "PROBE_SOURCE_DECLARED: convention — Generative Metrics’ own probe source address is a DECLARED self-report supplied as an input, never a measured observation, so it can never be read as evidence about a vendor’s crawler.", "VERIFICATION_NOT_PERFORMED: convention — this protocol records the state of the published identity material. It performs no match and no mismatch between a request and an address range, and its observation result is INDETERMINATE whenever a verification cannot be made. Every state reachable from Generative Metrics’ vantage is such a case.", "Only a payload hash, a bounded prefix count and a shape verdict are recorded for each vendor document; no document body and no address value leaves the retrieval phase.", "VANTAGE_LIMITED: convention — the vendor-published identity material is in hand for at least one audited actor, and the verification itself still cannot be performed from Generative Metrics’ vantage. The material is recorded so a site owner can use it directly with their own traffic measurements." ] }, { "observationId": "aec80bea-e212-4260-9e2a-17b11c1d345f", "protocolExecutionId": "b60ff6e7-c9dc-43d2-bfb2-eeca5d62c334", "kind": "CRAWLER_IDENTITY_VERIFICATION", "result": "NOT_OBSERVED", "evidenceClass": "DECLARED", "value": { "actorId": "google:google-cloudvertexbot", "actorProvider": "Google", "productToken": "Google-CloudVertexBot", "verificationState": "RANGES_UNAVAILABLE", "publishedRangesState": "NO_PUBLISHED_RANGES_FOR_ACTOR", "verificationPerformed": false, "vendorSystemsTookPart": false, "canVerifyVendorCrawler": false, "customerSideMeasurementInputAvailable": false, "declaredSourceId": null, "declaredSourceUrl": null, "documentRetrieved": null, "documentShape": null, "documentHttpStatus": null, "documentByteLength": null, "documentBodyTruncated": null, "documentPayloadSha256": null, "documentRetrievedAt": null, "documentFailureCause": null, "publishedPrefixCount": null, "publishedIpv4PrefixCount": null, "publishedIpv6PrefixCount": null, "probeSourceAddress": null, "probeSourceEvidenceClass": "DECLARED" }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:18.117Z", "limitations": [ "VANTAGE_LIMITATION: convention — Generative Metrics observes from its own network vantage with a single synthetic request sent from its own declared scan address. A request from our address can never confirm that traffic claiming a crawler name is that vendor’s crawler, because the vendor’s fleet uses its own address ranges, reverse DNS, edge signatures and script behaviour. No verification result is claimed, in either direction, by this protocol.", "PROBE_SOURCE_DECLARED: convention — Generative Metrics’ own probe source address is a DECLARED self-report supplied as an input, never a measured observation, so it can never be read as evidence about a vendor’s crawler.", "VERIFICATION_NOT_PERFORMED: convention — this protocol records the state of the published identity material. It performs no match and no mismatch between a request and an address range, and its observation result is INDETERMINATE whenever a verification cannot be made. Every state reachable from Generative Metrics’ vantage is such a case.", "Only a payload hash, a bounded prefix count and a shape verdict are recorded for each vendor document; no document body and no address value leaves the retrieval phase.", "VANTAGE_LIMITED: convention — the vendor-published identity material is in hand for at least one audited actor, and the verification itself still cannot be performed from Generative Metrics’ vantage. The material is recorded so a site owner can use it directly with their own traffic measurements." ] }, { "observationId": "2089b294-e1b6-426f-9c41-8696e95d0fb9", "protocolExecutionId": "b60ff6e7-c9dc-43d2-bfb2-eeca5d62c334", "kind": "CRAWLER_IDENTITY_VERIFICATION", "result": "NOT_OBSERVED", "evidenceClass": "DECLARED", "value": { "actorId": "google:google-gemininotebook", "actorProvider": "Google", "productToken": "Google-GeminiNotebook", "verificationState": "RANGES_UNAVAILABLE", "publishedRangesState": "NO_PUBLISHED_RANGES_FOR_ACTOR", "verificationPerformed": false, "vendorSystemsTookPart": false, "canVerifyVendorCrawler": false, "customerSideMeasurementInputAvailable": false, "declaredSourceId": null, "declaredSourceUrl": null, "documentRetrieved": null, "documentShape": null, "documentHttpStatus": null, "documentByteLength": null, "documentBodyTruncated": null, "documentPayloadSha256": null, "documentRetrievedAt": null, "documentFailureCause": null, "publishedPrefixCount": null, "publishedIpv4PrefixCount": null, "publishedIpv6PrefixCount": null, "probeSourceAddress": null, "probeSourceEvidenceClass": "DECLARED" }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:18.117Z", "limitations": [ "VANTAGE_LIMITATION: convention — Generative Metrics observes from its own network vantage with a single synthetic request sent from its own declared scan address. A request from our address can never confirm that traffic claiming a crawler name is that vendor’s crawler, because the vendor’s fleet uses its own address ranges, reverse DNS, edge signatures and script behaviour. No verification result is claimed, in either direction, by this protocol.", "PROBE_SOURCE_DECLARED: convention — Generative Metrics’ own probe source address is a DECLARED self-report supplied as an input, never a measured observation, so it can never be read as evidence about a vendor’s crawler.", "VERIFICATION_NOT_PERFORMED: convention — this protocol records the state of the published identity material. It performs no match and no mismatch between a request and an address range, and its observation result is INDETERMINATE whenever a verification cannot be made. Every state reachable from Generative Metrics’ vantage is such a case.", "Only a payload hash, a bounded prefix count and a shape verdict are recorded for each vendor document; no document body and no address value leaves the retrieval phase.", "VANTAGE_LIMITED: convention — the vendor-published identity material is in hand for at least one audited actor, and the verification itself still cannot be performed from Generative Metrics’ vantage. The material is recorded so a site owner can use it directly with their own traffic measurements." ] }, { "observationId": "8e55f462-b834-41a3-80c2-1d1b0a286652", "protocolExecutionId": "b60ff6e7-c9dc-43d2-bfb2-eeca5d62c334", "kind": "CRAWLER_IDENTITY_VERIFICATION", "result": "NOT_OBSERVED", "evidenceClass": "DECLARED", "value": { "actorId": "google:googleother", "actorProvider": "Google", "productToken": "GoogleOther", "verificationState": "RANGES_UNAVAILABLE", "publishedRangesState": "NO_PUBLISHED_RANGES_FOR_ACTOR", "verificationPerformed": false, "vendorSystemsTookPart": false, "canVerifyVendorCrawler": false, "customerSideMeasurementInputAvailable": false, "declaredSourceId": null, "declaredSourceUrl": null, "documentRetrieved": null, "documentShape": null, "documentHttpStatus": null, "documentByteLength": null, "documentBodyTruncated": null, "documentPayloadSha256": null, "documentRetrievedAt": null, "documentFailureCause": null, "publishedPrefixCount": null, "publishedIpv4PrefixCount": null, "publishedIpv6PrefixCount": null, "probeSourceAddress": null, "probeSourceEvidenceClass": "DECLARED" }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:18.117Z", "limitations": [ "VANTAGE_LIMITATION: convention — Generative Metrics observes from its own network vantage with a single synthetic request sent from its own declared scan address. A request from our address can never confirm that traffic claiming a crawler name is that vendor’s crawler, because the vendor’s fleet uses its own address ranges, reverse DNS, edge signatures and script behaviour. No verification result is claimed, in either direction, by this protocol.", "PROBE_SOURCE_DECLARED: convention — Generative Metrics’ own probe source address is a DECLARED self-report supplied as an input, never a measured observation, so it can never be read as evidence about a vendor’s crawler.", "VERIFICATION_NOT_PERFORMED: convention — this protocol records the state of the published identity material. It performs no match and no mismatch between a request and an address range, and its observation result is INDETERMINATE whenever a verification cannot be made. Every state reachable from Generative Metrics’ vantage is such a case.", "Only a payload hash, a bounded prefix count and a shape verdict are recorded for each vendor document; no document body and no address value leaves the retrieval phase.", "VANTAGE_LIMITED: convention — the vendor-published identity material is in hand for at least one audited actor, and the verification itself still cannot be performed from Generative Metrics’ vantage. The material is recorded so a site owner can use it directly with their own traffic measurements." ] }, { "observationId": "747e72b5-fd6b-473a-a0a8-dfbdf6f966eb", "protocolExecutionId": "b60ff6e7-c9dc-43d2-bfb2-eeca5d62c334", "kind": "CRAWLER_IDENTITY_VERIFICATION", "result": "NOT_OBSERVED", "evidenceClass": "DECLARED", "value": { "actorId": "firecrawl:firecrawlagent", "actorProvider": "Firecrawl", "productToken": "FirecrawlAgent", "verificationState": "RANGES_UNAVAILABLE", "publishedRangesState": "NO_PUBLISHED_RANGES_FOR_ACTOR", "verificationPerformed": false, "vendorSystemsTookPart": false, "canVerifyVendorCrawler": false, "customerSideMeasurementInputAvailable": false, "declaredSourceId": null, "declaredSourceUrl": null, "documentRetrieved": null, "documentShape": null, "documentHttpStatus": null, "documentByteLength": null, "documentBodyTruncated": null, "documentPayloadSha256": null, "documentRetrievedAt": null, "documentFailureCause": null, "publishedPrefixCount": null, "publishedIpv4PrefixCount": null, "publishedIpv6PrefixCount": null, "probeSourceAddress": null, "probeSourceEvidenceClass": "DECLARED" }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:18.117Z", "limitations": [ "VANTAGE_LIMITATION: convention — Generative Metrics observes from its own network vantage with a single synthetic request sent from its own declared scan address. A request from our address can never confirm that traffic claiming a crawler name is that vendor’s crawler, because the vendor’s fleet uses its own address ranges, reverse DNS, edge signatures and script behaviour. No verification result is claimed, in either direction, by this protocol.", "PROBE_SOURCE_DECLARED: convention — Generative Metrics’ own probe source address is a DECLARED self-report supplied as an input, never a measured observation, so it can never be read as evidence about a vendor’s crawler.", "VERIFICATION_NOT_PERFORMED: convention — this protocol records the state of the published identity material. It performs no match and no mismatch between a request and an address range, and its observation result is INDETERMINATE whenever a verification cannot be made. Every state reachable from Generative Metrics’ vantage is such a case.", "Only a payload hash, a bounded prefix count and a shape verdict are recorded for each vendor document; no document body and no address value leaves the retrieval phase.", "VANTAGE_LIMITED: convention — the vendor-published identity material is in hand for at least one audited actor, and the verification itself still cannot be performed from Generative Metrics’ vantage. The material is recorded so a site owner can use it directly with their own traffic measurements." ] } ], "limitations": [ "VANTAGE_LIMITATION: convention — Generative Metrics observes from its own network vantage with a single synthetic request sent from its own declared scan address. A request from our address can never confirm that traffic claiming a crawler name is that vendor’s crawler, because the vendor’s fleet uses its own address ranges, reverse DNS, edge signatures and script behaviour. No verification result is claimed, in either direction, by this protocol.", "PROBE_SOURCE_DECLARED: convention — Generative Metrics’ own probe source address is a DECLARED self-report supplied as an input, never a measured observation, so it can never be read as evidence about a vendor’s crawler.", "VENDOR_PUBLISHED_RANGES: convention — the vendor identity documents are retrieved out of band by a scheduled refresh; only a payload hash, a bounded entry count and a shape verdict are recorded, and no document body is stored in the evidence bundle.", "VERIFICATION_NOT_PERFORMED: convention — this protocol records the state of the published identity material. It performs no match and no mismatch between a request and an address range, and its observation result is INDETERMINATE whenever a verification cannot be made. Every state reachable from Generative Metrics’ vantage is such a case.", "CRAWLER_IDENTITY_REMEDIATION_EXCLUDED: convention — this execution carries no remediation and no verification-protocol reference, so it can never reach a VERIFIED outcome. A verified outcome would require a customer-side measurement input (server logs or edge telemetry) that does not exist in this protocol set.", "Only a payload hash, a bounded prefix count and a shape verdict are recorded for each vendor document; no document body and no address value leaves the retrieval phase.", "RANGES_UNAVAILABLE: convention — no retrieved copy of the vendor-published identity material was available to this audit (the out-of-band refresh cache was cold, absent, or the document could not be retrieved), so no availability fact is recorded for those actors." ] }, { "protocolExecutionId": "b4ffe5ea-ff39-41ef-aaa4-5807bbf8ef11", "protocolId": "content.structure", "protocolVersion": "1.0.0", "stage": "INTERPRETATION", "required": false, "actor": { "actorId": "gm:direct-http-probe", "provider": "Generative Metrics", "displayName": "Generative Metrics direct HTTP probe", "purpose": "UNSPECIFIED", "robotsPolicyApplicability": "NOT_APPLICABLE", "userAgent": "GenerativeMetrics/1.0 (+https://generativemetrics.com)" }, "startedAt": "2026-10-23T12:23:18.118Z", "finishedAt": "2026-10-23T12:23:18.118Z", "artifactRefs": [ "f0cb1e46-f740-49c5-8cbb-ab49da5046db", "554ed831-6d27-4c75-83a7-0eace8fc2eb8" ], "input": { "rawArtifactId": "f0cb1e46-f740-49c5-8cbb-ab49da5046db", "renderedArtifactId": "554ed831-6d27-4c75-83a7-0eace8fc2eb8", "basis": "RENDERED_DOCUMENT" }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "9763ed03-74fa-452b-bfe1-221008d45572", "protocolExecutionId": "b4ffe5ea-ff39-41ef-aaa4-5807bbf8ef11", "kind": "CONTENT_STRUCTURE_FACTS", "result": "OBSERVED", "evidenceClass": "DERIVED", "value": { "h1Count": 1, "h2Count": 23, "h3Count": 3, "otherHeadingCount": 1, "headingOrderViolations": 0, "listCount": 9, "tableCount": 0, "emptyHeadingCount": 0, "headingsWithoutContentCount": 1, "duplicateHeadingTextsCount": 0, "minSectionWords": 0 }, "artifactRefs": [ "f0cb1e46-f740-49c5-8cbb-ab49da5046db", "554ed831-6d27-4c75-83a7-0eace8fc2eb8" ], "producedAt": "2026-10-23T12:23:18.118Z", "limitations": [ "Counts are HTML tag counts in the rendered document; they measure structure, not text quality.", "A skipped heading level is counted only between adjacent heading tags in document order.", "JS-only pages are measured from the rendered DOM, so client-rendered structure is included.", "Fingerprint facts (empty headings, headings with no content beneath them, duplicated heading texts, fewest words under a heading) are divergence aids, not findings; section words count text in leaf paragraphs, list items, and table cells, each counted once." ] } ], "limitations": [ "Counts are HTML tag counts in the rendered document; they measure structure, not text quality.", "A skipped heading level is counted only between adjacent heading tags in document order.", "JS-only pages are measured from the rendered DOM, so client-rendered structure is included.", "Fingerprint facts (empty headings, headings with no content beneath them, duplicated heading texts, fewest words under a heading) are divergence aids, not findings; section words count text in leaf paragraphs, list items, and table cells, each counted once." ] }, { "protocolExecutionId": "1696b43e-9034-465b-bc3c-ab356bd4ed29", "protocolId": "metadata.signals", "protocolVersion": "1.0.0", "stage": "INTERPRETATION", "required": false, "actor": { "actorId": "gm:direct-http-probe", "provider": "Generative Metrics", "displayName": "Generative Metrics direct HTTP probe", "purpose": "UNSPECIFIED", "robotsPolicyApplicability": "NOT_APPLICABLE", "userAgent": "GenerativeMetrics/1.0 (+https://generativemetrics.com)" }, "startedAt": "2026-10-23T12:23:18.125Z", "finishedAt": "2026-10-23T12:23:18.125Z", "artifactRefs": [ "f0cb1e46-f740-49c5-8cbb-ab49da5046db", "554ed831-6d27-4c75-83a7-0eace8fc2eb8" ], "input": { "rawArtifactId": "f0cb1e46-f740-49c5-8cbb-ab49da5046db", "renderedArtifactId": "554ed831-6d27-4c75-83a7-0eace8fc2eb8", "renderedFinalUrl": "https://www.usa.gov/" }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "40def14b-5f89-4d7f-8ccf-2b2d0a82959c", "protocolExecutionId": "1696b43e-9034-465b-bc3c-ab356bd4ed29", "kind": "METADATA_SIGNAL_FACTS", "result": "OBSERVED", "evidenceClass": "DERIVED", "value": { "canonicalPresent": true, "canonicalSelfConsistent": true, "canonicalResolvedUrl": "https://www.usa.gov/", "internalLinkCount": 73, "externalLinkCount": 4 }, "artifactRefs": [ "f0cb1e46-f740-49c5-8cbb-ab49da5046db", "554ed831-6d27-4c75-83a7-0eace8fc2eb8" ], "producedAt": "2026-10-23T12:23:18.125Z", "limitations": [ "Canonical and anchor facts are read from the rendered document’s HTML tags only.", "The canonical URL is resolved at scan time against the response’s final URL; a redirect chain that changes between scans changes the resolved value, so the recorded URL is observed-at-scan-time.", "Internal links are anchors resolving to the same hostname with a leading \"www.\" ignored; other subdomains count as external.", "Anchor traversal is bounded at 5000 anchors; non-http(s) and unresolvable hrefs are not counted.", "A canonical URL pointing at a different address can be intentional (for example paginated or syndicated pages)." ] } ], "limitations": [ "Canonical and anchor facts are read from the rendered document’s HTML tags only.", "The canonical URL is resolved at scan time against the response’s final URL; a redirect chain that changes between scans changes the resolved value, so the recorded URL is observed-at-scan-time.", "Internal links are anchors resolving to the same hostname with a leading \"www.\" ignored; other subdomains count as external.", "Anchor traversal is bounded at 5000 anchors; non-http(s) and unresolvable hrefs are not counted.", "A canonical URL pointing at a different address can be intentional (for example paginated or syndicated pages)." ] }, { "protocolExecutionId": "05fd936a-66e7-4390-843e-f793d5bef7ce", "protocolId": "policy.opt-out-semantics", "protocolVersion": "1.0.0", "stage": "INTERPRETATION", "required": false, "actor": { "actorId": "gm:direct-http-probe", "provider": "Generative Metrics", "displayName": "Generative Metrics direct HTTP probe", "purpose": "UNSPECIFIED", "robotsPolicyApplicability": "NOT_APPLICABLE", "userAgent": "GenerativeMetrics/1.0 (+https://generativemetrics.com)" }, "startedAt": "2026-10-23T12:23:18.132Z", "input": { "rawArtifactId": "f0cb1e46-f740-49c5-8cbb-ab49da5046db", "renderedArtifactId": "554ed831-6d27-4c75-83a7-0eace8fc2eb8", "renderedHtmlCaptured": true, "tdmRepPath": "/.well-known/tdmrep.json" }, "failure": null, "finishedAt": "2026-10-23T12:23:19.353Z", "artifactRefs": [ "f0cb1e46-f740-49c5-8cbb-ab49da5046db", "554ed831-6d27-4c75-83a7-0eace8fc2eb8" ], "state": "SUCCEEDED", "observations": [ { "observationId": "b8f2711f-edc7-48cb-8b4f-03bfd5d545b8", "protocolExecutionId": "05fd936a-66e7-4390-843e-f793d5bef7ce", "kind": "OPT_OUT_META_ROBOTS", "result": "NOT_OBSERVED", "evidenceClass": "DERIVED", "value": { "present": false, "tokens": [], "unparseableTokenCount": 0, "conflictingDirectiveTagCount": 0, "directiveClass": null, "truncatedTokenList": false, "matrixVersion": null, "source": "RENDERED_DOCUMENT", "tagsObserved": 0 }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:19.353Z", "limitations": [ "Observed opt-out signals are policy signals only; provider compliance, access denial, training behavior, indexing, and citation impact are not inferred.", "Precedence convention: where both the X-Robots-Tag response header and a tag declare directives, systems that implement a precedence rule treat the X-Robots-Tag response header as controlling; this protocol records both sources separately and adjudicates nothing between them.", "within one source, conflicting meta-robots tokens resolve most-restrictive (the no-keyword tokens stand); sources are recorded separately and never merged; compliance by any engine is never inferred.", "Fixed opt-out directive grammar: all | index | follow | noindex | nofollow | noarchive | nosnippet | noimageindex | noodp | none | unavailable_after: | max-snippet: | max-image-preview:(none|standard|large) | max-video-preview:(-1|int) | noai | noimageai. Entries outside this grammar are counted as UNPARSEABLE_TOKEN_COUNT values and are never stored verbatim." ] }, { "observationId": "c7311877-6553-4719-8056-afd707c6c8db", "protocolExecutionId": "05fd936a-66e7-4390-843e-f793d5bef7ce", "kind": "OPT_OUT_X_ROBOTS_TAG", "result": "NOT_OBSERVED", "evidenceClass": "DIRECT", "value": { "present": false, "tokens": [], "unparseableTokenCount": 0, "conflictingDirectiveTagCount": 0, "directiveClass": null, "truncatedTokenList": false, "matrixVersion": null, "source": "RAW_HTTP_RESPONSE", "globalTokens": [], "scopedAgents": [] }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:19.353Z", "limitations": [ "Observed opt-out signals are policy signals only; provider compliance, access denial, training behavior, indexing, and citation impact are not inferred.", "Precedence convention: where both the X-Robots-Tag response header and a tag declare directives, systems that implement a precedence rule treat the X-Robots-Tag response header as controlling; this protocol records both sources separately and adjudicates nothing between them.", "within one source, conflicting meta-robots tokens resolve most-restrictive (the no-keyword tokens stand); sources are recorded separately and never merged; compliance by any engine is never inferred.", "Fixed opt-out directive grammar: all | index | follow | noindex | nofollow | noarchive | nosnippet | noimageindex | noodp | none | unavailable_after: | max-snippet: | max-image-preview:(none|standard|large) | max-video-preview:(-1|int) | noai | noimageai. Entries outside this grammar are counted as UNPARSEABLE_TOKEN_COUNT values and are never stored verbatim." ] }, { "observationId": "aa3ef067-3d18-4d32-9ff7-d2b5f46b5f0f", "protocolExecutionId": "05fd936a-66e7-4390-843e-f793d5bef7ce", "kind": "OPT_OUT_TDM_REP", "result": "NOT_OBSERVED", "evidenceClass": "DIRECT", "value": { "source": "WELL_KNOWN_TDMREP_JSON", "path": "/.well-known/tdmrep.json", "status": 404, "mediaType": "text/html", "byteLength": 0, "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bodyTruncated": false, "tdmReservation": "ABSENT", "invalidReason": null, "gateReason": null, "documentForm": null, "reserved": null, "invalidElementIndex": null }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:19.353Z", "limitations": [ "Observed opt-out signals are policy signals only; provider compliance, access denial, training behavior, indexing, and citation impact are not inferred.", "The site does not publish a TDM-Rep reservation document at /.well-known/tdmrep.json (HTTP 404); absence is not a reservation either way." ] } ], "limitations": [ "Observed opt-out signals are policy signals only; provider compliance, access denial, training behavior, indexing, and citation impact are not inferred.", "Precedence convention: where both the X-Robots-Tag response header and a tag declare directives, systems that implement a precedence rule treat the X-Robots-Tag response header as controlling; this protocol records both sources separately and adjudicates nothing between them.", "within one source, conflicting meta-robots tokens resolve most-restrictive (the no-keyword tokens stand); sources are recorded separately and never merged; compliance by any engine is never inferred.", "Fixed opt-out directive grammar: all | index | follow | noindex | nofollow | noarchive | nosnippet | noimageindex | noodp | none | unavailable_after: | max-snippet: | max-image-preview:(none|standard|large) | max-video-preview:(-1|int) | noai | noimageai. Entries outside this grammar are counted as UNPARSEABLE_TOKEN_COUNT values and are never stored verbatim." ] }, { "protocolExecutionId": "5907e2e6-63f8-4e5f-a827-f089cf4d3ea5", "protocolId": "policy.ai-txt", "protocolVersion": "1.0.0", "stage": "ACQUISITION", "required": false, "actor": { "actorId": "gm:ai-txt-probe", "provider": "Generative Metrics", "displayName": "Generative Metrics /ai.txt probe", "purpose": "UNSPECIFIED", "robotsPolicyApplicability": "NOT_APPLICABLE", "userAgent": "GenerativeMetrics/1.0 (+https://generativemetrics.com)" }, "state": "SUCCEEDED", "startedAt": "2026-10-23T12:23:19.354Z", "finishedAt": "2026-10-23T12:23:20.064Z", "artifactRefs": [ "f0cb1e46-f740-49c5-8cbb-ab49da5046db", "554ed831-6d27-4c75-83a7-0eace8fc2eb8" ], "input": { "targetUrl": "https://www.usa.gov/", "rawArtifactId": "f0cb1e46-f740-49c5-8cbb-ab49da5046db", "renderedArtifactId": "554ed831-6d27-4c75-83a7-0eace8fc2eb8", "aiTxtPath": "/ai.txt" }, "observations": [ { "observationId": "743c83bb-ad62-4702-9b17-fa45287c5bfb", "protocolExecutionId": "5907e2e6-63f8-4e5f-a827-f089cf4d3ea5", "kind": "AI_TXT_STATUS", "result": "NOT_OBSERVED", "evidenceClass": "DIRECT", "value": { "status": 404, "byteLength": 0, "mediaType": "text/html", "present": false, "titleObserved": null, "titleChars": null, "sectionCount": null, "valid": null, "truncated": false, "verdict": null }, "artifactRefs": [], "producedAt": "2026-10-23T12:23:20.064Z", "limitations": [ "The target does not publish a /ai.txt file.", "ai.txt is an IETF individual draft (draft-car-ai-txt-wellknown-00, June 2026) with no documented crawler adoption; this check records presence and shape only." ] } ], "failure": null, "limitations": [ "ai.txt is an IETF individual draft (draft-car-ai-txt-wellknown-00, June 2026) with no documented crawler adoption; this check records presence and shape only." ] }, { "protocolExecutionId": "b97b2724-c17f-444c-ac81-34581fd59cbb", "protocolId": "structured-data.depth", "protocolVersion": "1.0.0", "stage": "INTERPRETATION", "required": false, "actor": { "actorId": "gm:direct-http-probe", "provider": "Generative Metrics", "displayName": "Generative Metrics direct HTTP probe", "purpose": "UNSPECIFIED", "robotsPolicyApplicability": "NOT_APPLICABLE", "userAgent": "GenerativeMetrics/1.0 (+https://generativemetrics.com)" }, "startedAt": "2026-10-23T12:23:20.064Z", "finishedAt": "2026-10-23T12:23:20.064Z", "artifactRefs": [ "f0cb1e46-f740-49c5-8cbb-ab49da5046db", "554ed831-6d27-4c75-83a7-0eace8fc2eb8" ], "input": { "rawArtifactId": "f0cb1e46-f740-49c5-8cbb-ab49da5046db", "renderedArtifactId": "554ed831-6d27-4c75-83a7-0eace8fc2eb8", "renderedHtmlCaptured": true, "conventionVersion": "v1", "conventionCitedAt": "2026-10-02", "maxTypes": 20, "maxPropertyChecks": 200 }, "failure": null, "state": "SUCCEEDED", "observations": [ { "observationId": "318459f8-5bfc-408c-8acb-98b4aae03637", "protocolExecutionId": "b97b2724-c17f-444c-ac81-34581fd59cbb", "kind": "STRUCTURED_DATA_PROPERTY_DEPTH", "result": "OBSERVED", "evidenceClass": "DERIVED", "value": { "convention": { "name": "GM material-property convention", "version": "v1", "citedAt": "2026-10-02", "expectations": { "Article": [ "headline", "datePublished", "author" ], "BlogPosting": [ "headline", "datePublished", "author" ], "NewsArticle": [ "headline", "datePublished", "author" ], "Product": [ "name", "offers" ], "LocalBusiness": [ "name", "address", "telephone" ], "Organization": [ "name", "url" ], "Event": [ "name", "startDate", "location" ], "Recipe": [ "name", "recipeIngredient" ], "FAQPage": [ "mainEntity" ] } }, "checkedArm": "RAW_JSON_LD", "diagnostics": { "jsonLdBlocksFound": 2, "jsonLdBlocksParsed": 2, "jsonLdBlocksFailed": 0 }, "jsonLdSchemaCount": 2, "checkedTypes": [ { "type": "Organization", "missing": [ "name" ], "checked": [ "name", "url" ] } ], "typesWithMissingProperties": 1, "conventionTypesComplete": [], "conventionTypesAbsent": [ "Article", "BlogPosting", "Event", "FAQPage", "LocalBusiness", "NewsArticle", "Product", "Recipe" ], "observedOtherTypes": [ "WebSite" ], "caps": { "maxTypes": 20, "maxPropertyChecks": 200 }, "propertyChecksExecuted": 2, "typeChecksTruncated": false, "propertyChecksTruncated": false }, "artifactRefs": [ "f0cb1e46-f740-49c5-8cbb-ab49da5046db", "554ed831-6d27-4c75-83a7-0eace8fc2eb8" ], "producedAt": "2026-10-23T12:23:20.064Z", "limitations": [ "Property expectations are the Generative Metrics material-property convention (version v1, cited 2026-10-02); schema.org defines no required properties, and the same convention list applies on every site and every scan.", "The depth check reads the RAW HTML arm only; the browser-rendered representation is not property-checked.", "Property presence is recorded per type as booleans; property values, hashes, and page text are never stored.", "Only the raw JSON-LD graph is property-checked; microdata, RDFa, and OpenGraph-synthesized objects are recorded as counts only.", "Property absence under this convention is a counted fact, not a verdict input, and the convention defines no required properties for any type." ] } ], "limitations": [ "Property expectations are the Generative Metrics material-property convention (version v1, cited 2026-10-02); schema.org defines no required properties, and the same convention list applies on every site and every scan.", "The depth check reads the RAW HTML arm only; the browser-rendered representation is not property-checked.", "Property presence is recorded per type as booleans; property values, hashes, and page text are never stored.", "Only the raw JSON-LD graph is property-checked; microdata, RDFa, and OpenGraph-synthesized objects are recorded as counts only.", "Property absence under this convention is a counted fact, not a verdict input, and the convention defines no required properties for any type." ] }, { "protocolExecutionId": "1f89eec0-5bc6-4a2d-8b4c-f9494dddcb82", "protocolId": "extraction.pdf-text", "protocolVersion": "1.0.0", "stage": "INTERPRETATION", "required": false, "actor": { "actorId": "gm:direct-http-probe", "provider": "Generative Metrics", "displayName": "Generative Metrics direct HTTP probe", "purpose": "UNSPECIFIED", "robotsPolicyApplicability": "NOT_APPLICABLE", "userAgent": "GenerativeMetrics/1.0 (+https://generativemetrics.com)" }, "startedAt": "2026-10-23T12:23:20.069Z", "artifactRefs": [ "f0cb1e46-f740-49c5-8cbb-ab49da5046db" ], "input": { "rawArtifactId": "f0cb1e46-f740-49c5-8cbb-ab49da5046db", "rawContentType": "text/html", "pageCap": 200, "budgetMs": 10000, "extractorLoaderDefaulted": true }, "failure": null, "finishedAt": "2026-10-23T12:23:20.069Z", "state": "ABSTAINED", "observations": [], "limitations": [ "not a PDF target: the raw representation declared content type text/html; the extraction.pdf-text protocol applies only to application/pdf." ] }, { "protocolExecutionId": "d54a0e98-320b-4516-868a-cfafda0f6b03", "protocolId": "site.robots-coverage", "protocolVersion": "1.0.0", "stage": "INTERPRETATION", "required": false, "actor": { "actorId": "gm:direct-http-probe", "provider": "Generative Metrics", "displayName": "Generative Metrics direct HTTP probe", "purpose": "UNSPECIFIED", "robotsPolicyApplicability": "NOT_APPLICABLE", "userAgent": "GenerativeMetrics/1.0 (+https://generativemetrics.com)" }, "state": "SUCCEEDED", "startedAt": "2026-10-23T12:23:20.069Z", "finishedAt": "2026-10-23T12:23:20.069Z", "artifactRefs": [ "3febc7be-434b-4720-889e-7f04cb6d7b2b", "69a414b7-f69c-44f6-8c02-289242626038" ], "input": { "sitemapProtocolExecutionId": "3b289399-4448-4493-96ca-59939a5f9bb7", "robotsPolicyExecutionIds": [ "0026771b-82bc-4369-9936-749a5fae813c", "0d937fef-7df4-4208-bfc0-f7f89407f3a9", "0e5f38c9-50cd-43a9-8acc-dee6ab76d14b", "16670dce-2151-4eaf-b483-56233162c40e", "16d4fbe6-5463-493a-90a8-6029bd56a88e", "1818f60b-7b1a-4bfc-8b0b-f1e8ed9964d3", "2b11bb86-d1cb-4911-89ff-c041e2a450c2", "2e9cd75d-5dbd-4a8e-a77f-86248ff2cc6a", "3abb3472-3a4a-42da-ab1a-09b40fa23120", "4f576ddf-ccb7-441d-809a-85309defcfb9", "54adf426-1696-4db9-ad8a-73d3bc855210", "5ede9dcc-efcd-44a2-ae54-b02222a5e6c8", "60c2dd5d-b305-40ab-8373-9e13093dec78", "6bbf2bc9-8d6f-4e6c-ba6b-4f5edaac5597", "6c0784cc-c90a-49c0-b371-f927852afc46", "7032f20b-b9ec-44b7-a145-2a65ec3c7928", "7561f01a-5b2c-4e8b-ab62-c04fe7d3a31f", "7f38ea6a-cc04-46de-9308-8796af354d4a", "8b922115-563b-4e3b-9b56-3ec059b930e0", "95b10d45-cc88-4b69-ac2f-724ab90424b3", "a32b95a1-d580-4368-914f-06826588f0be", "ae477e6d-7a2a-4f0b-9cf1-f2482be7bbf0", "b4e7dfbd-09ca-4386-ae02-2a245212cdf4", "b5f96514-6b4f-4a76-a8ba-ead6aeb8ec98", "ccd0c17e-ba26-4651-b1b3-a13305542763", "debfb47a-a66b-43c9-97d2-212ea94793e3", "e88688c8-e9bb-4ac5-bc08-81ba6b7c0e3a" ], "requiredActorIds": [ "anthropic:claude-searchbot", "google:googlebot", "openai:oai-searchbot", "perplexity:perplexitybot" ], "inventoryCap": 5000, "exampleCap": 20 }, "observations": [ { "observationId": "fd7c51fd-6213-40bd-bea5-7173c46b6fb7", "protocolExecutionId": "d54a0e98-320b-4516-868a-cfafda0f6b03", "kind": "SITE_ROBOTS_COVERAGE", "result": "OBSERVED", "evidenceClass": "DERIVED", "value": { "denominator": { "evaluatedUrls": 1526, "capturedTotal": 1526, "listTruncated": false }, "actors": [ { "actorId": "allenai:ai2bot", "group": "*", "evaluatedCount": 0, "disallowedCount": 0, "matchedRuleCounts": {}, "exampleDisallowedUrls": [], "excludedCount": 1526 }, { "actorId": "amazon:amazonbot", "group": "*", "evaluatedCount": 1526, "disallowedCount": 0, "matchedRuleCounts": {}, "exampleDisallowedUrls": [], "excludedCount": 0 }, { "actorId": "anthropic:claude-searchbot", "group": "*", "evaluatedCount": 1526, "disallowedCount": 0, "matchedRuleCounts": {}, "exampleDisallowedUrls": [], "excludedCount": 0 }, { "actorId": "anthropic:claude-user", "group": "*", "evaluatedCount": 1526, "disallowedCount": 0, "matchedRuleCounts": {}, "exampleDisallowedUrls": [], "excludedCount": 0 }, { "actorId": "anthropic:claudebot", "group": "*", "evaluatedCount": 1526, "disallowedCount": 0, "matchedRuleCounts": {}, "exampleDisallowedUrls": [], "excludedCount": 0 }, { "actorId": "apple:applebot", "group": "*", "evaluatedCount": 1526, "disallowedCount": 0, "matchedRuleCounts": {}, "exampleDisallowedUrls": [], "excludedCount": 0 }, { "actorId": "apple:applebot-extended", "group": "*", "evaluatedCount": 0, "disallowedCount": 0, "matchedRuleCounts": {}, "exampleDisallowedUrls": [], "excludedCount": 1526 }, { "actorId": "commoncrawl:ccbot", "group": "*", "evaluatedCount": 1526, "disallowedCount": 0, "matchedRuleCounts": {}, "exampleDisallowedUrls": [], "excludedCount": 0 }, { "actorId": "diffbot:diffbot", "group": "*", "evaluatedCount": 1526, "disallowedCount": 0, "matchedRuleCounts": {}, "exampleDisallowedUrls": [], "excludedCount": 0 }, { "actorId": "duckduckgo:duckassistbot", "group": "*", "evaluatedCount": 1526, "disallowedCount": 0, "matchedRuleCounts": {}, "exampleDisallowedUrls": [], "excludedCount": 0 }, { "actorId": "firecrawl:firecrawlagent", "group": "*", "evaluatedCount": 1526, "disallowedCount": 0, "matchedRuleCounts": {}, "exampleDisallowedUrls": [], "excludedCount": 0 }, { "actorId": "google:google-agent", "group": "*", "evaluatedCount": 0, "disallowedCount": 0, "matchedRuleCounts": {}, "exampleDisallowedUrls": [], "excludedCount": 1526 }, { "actorId": "google:google-cloudvertexbot", "group": "*", "evaluatedCount": 1526, "disallowedCount": 0, "matchedRuleCounts": {}, "exampleDisallowedUrls": [], "excludedCount": 0 }, { "actorId": "google:google-extended", "group": "*", "evaluatedCount": 0, "disallowedCount": 0, "matchedRuleCounts": {}, "exampleDisallowedUrls": [], "excludedCount": 1526 }, { "actorId": "google:google-gemininotebook", "group": "*", "evaluatedCount": 0, "disallowedCount": 0, "matchedRuleCounts": {}, "exampleDisallowedUrls": [], "excludedCount": 1526 }, { "actorId": "google:googlebot", "group": "*", "evaluatedCount": 1526, "disallowedCount": 0, "matchedRuleCounts": {}, "exampleDisallowedUrls": [], "excludedCount": 0 }, { "actorId": "google:googleother", "group": "*", "evaluatedCount": 1526, "disallowedCount": 0, "matchedRuleCounts": {}, "exampleDisallowedUrls": [], "excludedCount": 0 }, { "actorId": "hive:imagesiftbot", "group": "*", "evaluatedCount": 1526, "disallowedCount": 0, "matchedRuleCounts": {}, "exampleDisallowedUrls": [], "excludedCount": 0 }, { "actorId": "kagi:kagibot", "group": "*", "evaluatedCount": 1526, "disallowedCount": 0, "matchedRuleCounts": {}, "exampleDisallowedUrls": [], "excludedCount": 0 }, { "actorId": "linkup:linkupbot", "group": "*", "evaluatedCount": 1526, "disallowedCount": 0, "matchedRuleCounts": {}, "exampleDisallowedUrls": [], "excludedCount": 0 }, { "actorId": "meta:meta-externalagent", "group": "*", "evaluatedCount": 1526, "disallowedCount": 0, "matchedRuleCounts": {}, "exampleDisallowedUrls": [], "excludedCount": 0 }, { "actorId": "meta:meta-externalfetcher", "group": "*", "evaluatedCount": 0, "disallowedCount": 0, "matchedRuleCounts": {}, "exampleDisallowedUrls": [], "excludedCount": 1526 }, { "actorId": "microsoft:bingbot", "group": "*", "evaluatedCount": 0, "disallowedCount": 0, "matchedRuleCounts": {}, "exampleDisallowedUrls": [], "excludedCount": 1526 }, { "actorId": "mistralai:mistralai-user", "group": "*", "evaluatedCount": 1526, "disallowedCount": 0, "matchedRuleCounts": {}, "exampleDisallowedUrls": [], "excludedCount": 0 }, { "actorId": "openai:chatgpt-user", "group": "*", "evaluatedCount": 0, "disallowedCount": 0, "matchedRuleCounts": {}, "exampleDisallowedUrls": [], "excludedCount": 1526 }, { "actorId": "openai:gptbot", "group": "*", "evaluatedCount": 1526, "disallowedCount": 0, "matchedRuleCounts": {}, "exampleDisallowedUrls": [], "excludedCount": 0 }, { "actorId": "openai:oai-adsbot", "group": "*", "evaluatedCount": 0, "disallowedCount": 0, "matchedRuleCounts": {}, "exampleDisallowedUrls": [], "excludedCount": 1526 }, { "actorId": "openai:oai-searchbot", "group": "*", "evaluatedCount": 1526, "disallowedCount": 0, "matchedRuleCounts": {}, "exampleDisallowedUrls": [], "excludedCount": 0 }, { "actorId": "perplexity:perplexity-user", "group": "*", "evaluatedCount": 0, "disallowedCount": 0, "matchedRuleCounts": {}, "exampleDisallowedUrls": [], "excludedCount": 1526 }, { "actorId": "perplexity:perplexitybot", "group": "*", "evaluatedCount": 1526, "disallowedCount": 0, "matchedRuleCounts": {}, "exampleDisallowedUrls": [], "excludedCount": 0 }, { "actorId": "webzio:omgili", "group": "*", "evaluatedCount": 1526, "disallowedCount": 0, "matchedRuleCounts": {}, "exampleDisallowedUrls": [], "excludedCount": 0 }, { "actorId": "xai:xai-searchbot", "group": "*", "evaluatedCount": 0, "disallowedCount": 0, "matchedRuleCounts": {}, "exampleDisallowedUrls": [], "excludedCount": 1526 } ] }, "artifactRefs": [ "3febc7be-434b-4720-889e-7f04cb6d7b2b", "69a414b7-f69c-44f6-8c02-289242626038" ], "producedAt": "2026-10-23T12:23:20.069Z", "limitations": [ "Only the captured sitemap document was evaluated; sitemaps declared in robots.txt and child sitemaps listed in a sitemap index were not fetched.", "Loc entries for context-dependent or control-token actors are excluded from the per-actor evaluated and disallowed counts (disclosed as excludedCount): their directives are reported without inferring that robots.txt governs actual access.", "Disallow counts, matched-rule tokens, and example URLs are execution facts only; no compliance, access, indexing, or citation effect is inferred. Example disallowed URLs are recorded inside the sealed evidence bundle and are erased with it." ] } ], "failure": null, "limitations": [ "Only the captured sitemap document was evaluated; sitemaps declared in robots.txt and child sitemaps listed in a sitemap index were not fetched.", "Loc entries for context-dependent or control-token actors are excluded from the per-actor evaluated and disallowed counts (disclosed as excludedCount): their directives are reported without inferring that robots.txt governs actual access.", "Disallow counts, matched-rule tokens, and example URLs are execution facts only; no compliance, access, indexing, or citation effect is inferred. Example disallowed URLs are recorded inside the sealed evidence bundle and are erased with it." ] }, { "protocolExecutionId": "6bcf2acd-1f2a-45ae-8573-36bb3a70b428", "protocolId": "identity.wba-directory", "protocolVersion": "1.0.0", "stage": "ACQUISITION", "required": false, "actor": { "actorId": "gm:wba-directory-probe", "provider": "Generative Metrics", "displayName": "Generative Metrics Web Bot Auth directory probe", "purpose": "UNSPECIFIED", "robotsPolicyApplicability": "NOT_APPLICABLE", "userAgent": "GenerativeMetrics/1.0 (+https://generativemetrics.com)" }, "state": "NOT_APPLICABLE", "startedAt": "2026-10-23T12:23:23.672Z", "finishedAt": "2026-10-23T12:23:23.672Z", "artifactRefs": [], "input": { "directoryUrl": "https://www.usa.gov/.well-known/http-message-signatures-directory", "path": "/.well-known/http-message-signatures-directory", "coveredActors": [], "timeoutMs": 10000, "maxBodyBytes": 262144, "shapeRule": "VALID_SHAPE when the captured body parses as JSON and is an object with an array \"subscribers\" or \"keys\" member, or a non-empty array of objects; everything else retrievable is UNEXPECTED_SHAPE; NOT_RETRIEVED means no body was received.", "keyCountRule": "keyCount is the summed length of the \"keys\" arrays (per subscriber or top-level), or the element count of a bare array, capped at 4096; null unless the shape verdict is VALID_SHAPE." }, "observations": [], "failure": null, "limitations": [ "No covered actor publishes a Web Bot Auth key directory; the directory endpoint was not fetched.", "This records that a Web Bot Auth key-directory endpoint is published and reachable; it is never a verification of any request’s signature, of any traffic signing, or of any actor identity." ] }, { "protocolExecutionId": "027cafcc-5b0e-4049-a013-390ca21fc936", "protocolId": "policy.aipref", "protocolVersion": "1.0.0", "stage": "POLICY", "required": false, "actor": { "actorId": "gm:aipref-probe", "provider": "Generative Metrics", "displayName": "Generative Metrics AIPREF probe", "purpose": "UNSPECIFIED", "robotsPolicyApplicability": "NOT_APPLICABLE", "userAgent": "GenerativeMetrics/1.0 (+https://generativemetrics.com)" }, "state": "NOT_APPLICABLE", "startedAt": "2026-10-23T12:23:23.672Z", "finishedAt": "2026-10-23T12:23:23.672Z", "artifactRefs": [], "input": { "aiprefUrl": "https://www.usa.gov/.well-known/aipref", "path": "/.well-known/aipref", "robotsSignal": "ALLOWS", "tdmRepSignal": "NOT_MEASURED", "maxBodyBytes": 262144, "pinnedVocabulary": "vocab-08", "draftCited": "2026-06", "grammarRule": "The AIPREF document is valid when it parses as a JSON object whose member values are each either one of the pinned tokens (allow, restricted, closed) or a one-level-deep object of such tokens; anything else invalidates the document.", "conflictRule": "An axis conflicts exactly when at least two measured stances among robots, AIPREF and TDM-Rep differ; NO_POLICY and NOT_MEASURED carry no stance and never create a conflict." }, "observations": [], "failure": null, "limitations": [ "The site does not publish an AIPREF document at /.well-known/aipref.", "AIPREF is an experimental IETF draft signal; this record is parsed against the pinned AIPREF draft vocabulary vocab-08, cited 2026-06.", "The AIPREF document is a declared preference signal only; no provider compliance, access denial, training behavior, indexing, or citation impact is inferred.", "Cross-signal conflicts with robots.txt and TDM-Rep are enumerated mechanically; no signal is preferred and no conflict finding is emitted." ] }, { "protocolExecutionId": "4332aba1-63fd-4a5d-92f9-e1ef2756343f", "protocolId": "observation.cdn-default-block", "protocolVersion": "1.0.0", "stage": "INTERPRETATION", "required": false, "actor": { "actorId": "gm:direct-http-probe", "provider": "Generative Metrics", "displayName": "Generative Metrics direct HTTP probe", "purpose": "UNSPECIFIED", "robotsPolicyApplicability": "NOT_APPLICABLE", "userAgent": "GenerativeMetrics/1.0 (+https://generativemetrics.com)" }, "state": "NOT_APPLICABLE", "startedAt": "2026-10-23T12:23:24.510Z", "finishedAt": "2026-10-23T12:23:24.510Z", "artifactRefs": [], "input": { "actorAccessProtocolId": "access.actor-probe", "denialClassification": "ACCESS_BLOCKED", "matchRule": "A match is a sealed access.actor-probe ACTOR_ACCESS_RESULT whose classification field is ACCESS_BLOCKED or ACCESS_GATE, paired with a vendor entry whose published documentation names this denial class as a default behavior. The classification is never re-derived from the HTTP status, and no other classification matches.", "documentationTable": [ { "vendor": "cloudflare", "denialClass": "ai-crawler-default-block", "sourceCitation": "https://blog.cloudflare.com/content-independence-day-ai-options/", "citationDate": "2026-10-02" } ], "serverHeader": null }, "observations": [], "failure": null, "limitations": [ "No sealed access.actor-probe denial matched the documented CDN default-block class; the documentation table is recorded without a match.", "A match is a sealed access.actor-probe ACTOR_ACCESS_RESULT whose classification field is ACCESS_BLOCKED or ACCESS_GATE, paired with a vendor entry whose published documentation names this denial class as a default behavior. The classification is never re-derived from the HTTP status, and no other classification matches.", "The sealed probe record identifies no CDN (probe observations seal no response headers), so the vendor attribution comes from the vendor’s published documentation of this denial class, never from the scan.", "The CDN vendor documents this block class as a default behavior; whether it applies to this site is not measured, and no zone configuration is asserted." ] } ], "findings": [ { "findingId": "bc267b86-99e7-4b2a-8c79-369c30005bcf", "code": "STRUCTURED_DATA_ORGANIZATION_IDENTITY_INCOMPLETE", "findingClass": "PRODUCTION", "severity": "INFO", "state": "OPEN", "title": "The raw Organization structured data is missing identity fields", "summary": "An Organization node was observed in the raw response, and the bounded identity check did not observe: name. This is an informational completeness observation, not a defect by itself.", "protocolExecutionRefs": [ "0bc319bd-5095-428a-a132-e09d78243f6e" ], "observationRefs": [ "63c8adfd-f2c0-4cd4-81c8-708d04993542" ], "remediationRefs": [], "limitations": [ "Completeness checks cover raw-versus-rendered schema type presence and three bounded Organization identity fields only.", "Organization identity fields are recorded as booleans; property values are not stored.", "The absence of an Organization node is not automatically a defect." ] }, { "findingId": "8e842bde-5901-43cd-9da5-82326d734910", "code": "LLMS_TXT_NOT_FOUND", "findingClass": "PRODUCTION", "severity": "INFO", "state": "OPEN", "title": "The site does not publish a /llms.txt file", "summary": "The observed site does not publish a /llms.txt file. This is an informational optimization opportunity and does not indicate that AI systems cannot access, understand, or cite the site.", "protocolExecutionRefs": [ "96be7182-8825-40b5-acf6-c79fc27d46bb" ], "observationRefs": [ "05c0499e-4df0-48d0-b74b-fa1e57f074ea" ], "remediationRefs": [ "9ec7e009-cbf1-451c-801a-b3dbf0d62e7c" ], "limitations": [ "The target does not publish a /llms.txt file." ] }, { "findingId": "15175d2a-d6a1-446a-85cd-c77d891b094c", "code": "CONTENT_NEGOTIATION_MARKDOWN_NOT_SERVED", "findingClass": "PRODUCTION", "severity": "INFO", "state": "OPEN", "title": "The origin did not serve a markdown representation", "summary": "A request that preferred `text/markdown` was answered with text/html (HTTP 200) and no markdown alternate was declared. Serving a markdown representation is an agent-friendly convention rather than a requirement; this is informational and does not indicate that AI systems cannot access, understand, or cite the page, and provider consumption is not inferred.", "protocolExecutionRefs": [ "51f482d2-32e9-4bf7-b264-88ca529dda73" ], "observationRefs": [ "dc74ad6f-2b6c-4dbb-8fb1-3ec06f848f03" ], "remediationRefs": [], "limitations": [ "The request asked for `Accept: text/markdown` and the origin answered HTTP 200 with text/html.", "CONTENT_NEGOTIATION_CONVENTION: convention — serving a markdown representation is a publishing convention, not a requirement; this records what was requested and what came back and never infers how a provider consumes content.", "Only the HTTP status, media type, bounded byte length, truncation flag and markup-prefix boolean are recorded; no response body is retained.", "Markdown detection uses the declared media type plus a 512-character body prefix check, so a mislabelled representation is recorded as declared.", "The response body was read with a 262144-byte bound.", "Only a Link header entry with rel=\"alternate\" and a markdown media type is recognised as the origin advertising an alternate markdown representation." ] }, { "findingId": "69cf8839-5d5b-4938-8316-a7d682446fdc", "code": "CRAWLER_IDENTITY_NOT_VERIFIABLE", "findingClass": "PRODUCTION", "severity": "INFO", "state": "OPEN", "title": "Crawler identity was not verified by this scan", "summary": "Vendors publish machine-readable identity material for their crawlers (address ranges from OpenAI, Perplexity and Anthropic, and reverse-DNS and crawler documentation from Google), and no retrieved copy of that material was recorded in this bundle. For this audit, no retrieved copy of the vendors’ published identity material was available. A synthetic request sent from Generative Metrics’ own declared scan address can never confirm that traffic claiming a crawler name is that vendor’s crawler, because the vendor’s fleet uses its own address ranges, reverse DNS, edge signatures and script behaviour. Nothing in this finding says that any crawler did or did not visit the audited site. Verifying that requires a measurement only the site owner holds: server logs or edge telemetry recorded at the origin, which no external scan can substitute for. This is reported for information only: it never blocks the audit, never changes the verdict, and it carries no remediation.", "protocolExecutionRefs": [ "b60ff6e7-c9dc-43d2-bfb2-eeca5d62c334" ], "observationRefs": [ "fa869f4a-b73c-4ef7-87a1-939b45146c20", "13e6e6f3-0d6b-4e0c-a56e-b3dfc94789e4", "61fc8799-be24-4e41-a47f-84e3e2c1a596", "613c2fc8-6959-469c-9a85-80a47f7a0d98", "5b01e7cc-6f32-4ccb-9aff-2f5962ef3b44", "4411197a-ab52-4291-b9c0-f64d4018703b", "5c1f1745-85b2-409a-8af0-36660a643f21", "3ae4ed25-d971-4994-ac47-877eded7706e", "9fb467c2-f85c-4d24-a7d6-d78335a8285a", "8e904e6c-4cc6-4f01-bd2e-b057279c0bd4", "8f6c72d1-a2de-47a3-893f-286347be9a82", "8cd10223-f172-4eea-90a5-b900efe829cd", "d96a7587-141c-4ba5-9d5f-aa7dce902810", "3d425418-1bde-4cd5-90c3-49b195a9b360", "a980364a-90c0-4ef6-b870-0655425cc85f", "bbfe07cf-a5ef-40c8-bf1f-53a700a6dad0", "6f1e5ec6-f777-4a69-bff6-45f93c94c657", "34b7f0ac-e50f-4941-b117-5430c2ce4a03", "f5c8c18e-ac0e-4b50-9a4f-63957ee157c4", "39fc53e9-dc7e-4119-81d0-dccfcb929db3", "eecac4b0-a2a5-4f04-be3b-693d04a321f3", "8844f1d6-523f-40fe-85aa-88fb409828fb", "fedb72c2-1cf6-4bae-886c-f712934f2e93", "aec80bea-e212-4260-9e2a-17b11c1d345f", "2089b294-e1b6-426f-9c41-8696e95d0fb9", "8e55f462-b834-41a3-80c2-1d1b0a286652", "747e72b5-fd6b-473a-a0a8-dfbdf6f966eb" ], "remediationRefs": [], "limitations": [ "VANTAGE_LIMITATION: convention — Generative Metrics observes from its own network vantage with a single synthetic request sent from its own declared scan address. A request from our address can never confirm that traffic claiming a crawler name is that vendor’s crawler, because the vendor’s fleet uses its own address ranges, reverse DNS, edge signatures and script behaviour. No verification result is claimed, in either direction, by this protocol.", "PROBE_SOURCE_DECLARED: convention — Generative Metrics’ own probe source address is a DECLARED self-report supplied as an input, never a measured observation, so it can never be read as evidence about a vendor’s crawler.", "VERIFICATION_NOT_PERFORMED: convention — this protocol records the state of the published identity material. It performs no match and no mismatch between a request and an address range, and its observation result is INDETERMINATE whenever a verification cannot be made. Every state reachable from Generative Metrics’ vantage is such a case.", "Only a payload hash, a bounded prefix count and a shape verdict are recorded for each vendor document; no document body and no address value leaves the retrieval phase.", "VANTAGE_LIMITED: convention — the vendor-published identity material is in hand for at least one audited actor, and the verification itself still cannot be performed from Generative Metrics’ vantage. The material is recorded so a site owner can use it directly with their own traffic measurements." ] }, { "findingId": "ed6a9cf4-0dd9-4998-8940-883a44626cf4", "code": "STRUCTURED_DATA_PROPERTY_MISSING", "findingClass": "PRODUCTION", "severity": "WARNING", "state": "OPEN", "title": "Structured data is missing convention properties", "summary": "Raw structured data on this page omitted 1 property(ies) the Generative Metrics material-property convention (v1, cited 2026-10-02) enumerates for the types the page presents: the page presents Organization and the Generative Metrics material-property convention (v1) expects name, url — the check did not observe name. Schema.org defines no required properties — this is a counted fact against the printed convention, for review.", "protocolExecutionRefs": [ "b97b2724-c17f-444c-ac81-34581fd59cbb" ], "observationRefs": [ "318459f8-5bfc-408c-8acb-98b4aae03637" ], "remediationRefs": [ "c2353602-c900-458b-ac41-d41a62725206" ], "limitations": [ "Property expectations are the Generative Metrics material-property convention (version v1, cited 2026-10-02); schema.org defines no required properties, and the same convention list applies on every site and every scan.", "The depth check reads the RAW HTML arm only; the browser-rendered representation is not property-checked.", "Property presence is recorded per type as booleans; property values, hashes, and page text are never stored.", "Only the raw JSON-LD graph is property-checked; microdata, RDFa, and OpenGraph-synthesized objects are recorded as counts only.", "Property absence under this convention is a counted fact, not a verdict input, and the convention defines no required properties for any type." ] }, { "findingId": "1ee98acd-11f2-4bec-92b2-4587c6a6cc52", "code": "SITEMAP_ROBOTS_COVERAGE_FACTS", "findingClass": "PRODUCTION", "severity": "INFO", "state": "OPEN", "title": "The robots policy was evaluated over the captured sitemap inventory", "summary": "The robots policy was evaluated over 1526 of 1526 loc entries in the captured sitemap document. Per-actor disallow counts are recorded as execution facts; no access, compliance, indexing, or citation effect is inferred.", "protocolExecutionRefs": [ "d54a0e98-320b-4516-868a-cfafda0f6b03" ], "observationRefs": [ "fd7c51fd-6213-40bd-bea5-7173c46b6fb7" ], "remediationRefs": [], "limitations": [ "Only the captured sitemap document was evaluated; sitemaps declared in robots.txt and child sitemaps listed in a sitemap index were not fetched.", "Loc entries for context-dependent or control-token actors are excluded from the per-actor evaluated and disallowed counts (disclosed as excludedCount): their directives are reported without inferring that robots.txt governs actual access.", "Disallow counts, matched-rule tokens, and example URLs are execution facts only; no compliance, access, indexing, or citation effect is inferred. Example disallowed URLs are recorded inside the sealed evidence bundle and are erased with it." ] } ], "remediations": [ { "remediationId": "9ec7e009-cbf1-451c-801a-b3dbf0d62e7c", "code": "PUBLISH_LLMS_TXT", "title": "Publish a /llms.txt file", "summary": "Consider publishing a /llms.txt file describing the site for AI agents. Absence does not prove an access, understanding, or citation problem; this is an advisory optimization.", "findingRefs": [ "8e842bde-5901-43cd-9da5-82326d734910" ], "actions": [ "Create a /llms.txt file at the site origin with a valid H1-style title heading.", "Ensure the body is at least the required minimum size beneath the title.", "Describe what information you intend to share with AI agents without exposing unpublished content." ], "verificationProtocolIds": [ "acquisition.llms-txt" ], "expectedOutcome": { "present": true, "valid": true } }, { "remediationId": "c2353602-c900-458b-ac41-d41a62725206", "code": "ADD_CONVENTION_PROPERTIES", "title": "Add the convention properties to the observed structured data", "summary": "The page presents Organization and the Generative Metrics material-property convention (v1) expects name, url. Adding the convention properties makes the structured data machine-checkable against that convention.", "findingRefs": [ "ed6a9cf4-0dd9-4998-8940-883a44626cf4" ], "actions": [ "Add the unobserved convention properties (name) to the Organization nodes in the raw JSON-LD response.", "Keep the structured data in the initial HTML response so machine readers observe it without JavaScript.", "Re-run the audit to record the updated property-presence facts." ], "verificationProtocolIds": [ "structured-data.depth" ], "expectedOutcome": { "typesWithMissingProperties": 0, "convention": "v1 (2026-10-02)" } } ], "verifications": [], "limitations": [ "This profile tests documented public discovery controls; it does not prove indexing, retrieval, ranking, citation, training inclusion, or model use.", "User-initiated agents are informational because provider behavior may be context dependent.", "Informational actors never create blockers and are never required; an unmeasured informational actor is reported rather than inferred." ], "schemaVersion": "gm-evidence-bundle/1.0.0", "runState": "COMPLETE", "verdict": "PASS", "bundleSha256": "45397e9790fd90b2e13dcbe1f4a3d2817c04c1d9584bc7b07be0765cc830c37a" }