aoai_checklist.en.json

{
    "items": [
        {
            "category": "Responsible AI",
            "subcategory": "Metaprompting",
            "text": "Follow Metaprompting guardrails for resonsible AI",
            "waf": "Operational Excellence",
            "service": "Azure OpenAI",
            "guid": "a85b86ad-884f-48e3-9273-4b875ba18f10",
            "id": "AOAI.1",
            "severity": "High",
            "link": "https://learn.microsoft.com/azure/ai-services/openai/concepts/system-message#define-additional-safety-and-behavioral-guardrails"
        },
        {
            "category": "Operations Management",
            "subcategory": "Load Balancing",
            "text": "Consider Gateway patterns with APIM  or solutions like AI central for better rate limiting, load balancing, authentication and logging",
            "waf": "Operational Excellence",
            "service": "Azure OpenAI",
            "guid": "d4391898-cd28-48be-b6b1-7cb8245451e1",
            "id": "AOAI.10",
            "severity": "High",
            "link": "https://github.com/Azure-Samples/AI-Gateway"
        },
        {
            "category": "Operations Management",
            "subcategory": "Monitoring",
            "text": "Enable monitoring for your AOAI instances",
            "waf": "Operational Excellence",
            "service": "Azure OpenAI",
            "guid": "aed3453a-ec72-4392-97a1-52d6cc5e4029",
            "id": "AOAI.11",
            "severity": "High",
            "link": "https://techcommunity.microsoft.com/t5/fasttrack-for-azure/azure-openai-insights-monitoring-ai-with-confidence/ba-p/4026850"
        },
        {
            "category": "Operations Management",
            "subcategory": "Alerts",
            "text": "Create alerts to notify teams of events such as an entry in the activity log created by an action performed on the resource, such as regenerating its subscription keys or a metric threshold such as the number of errors exceeding 10 in an hour",
            "waf": "Operational Excellence",
            "service": "Azure OpenAI",
            "guid": "697cb391-ed16-4b2d-886f-0a0241addde6",
            "id": "AOAI.12",
            "severity": "High",
            "link": "https://learn.microsoft.com/azure/ai-services/openai/how-to/monitoring#set-up-alerts"
        },
        {
            "category": "Operations Management",
            "subcategory": "Monitoring",
            "text": "Monitor token usage to prevent service disruptions due to capacity",
            "waf": "Operational Excellence",
            "service": "Azure OpenAI",
            "guid": "8a477cde-b486-41bc-9bc1-0ae66e25d4d5",
            "id": "AOAI.13",
            "severity": "High",
            "link": "https://learn.microsoft.com/azure/ai-services/openai/how-to/monitoring"
        },
        {
            "category": "Operations Management",
            "subcategory": "Observability",
            "text": "observe metrics like processed inference tokens, generated completion tokens monitor for rate limit",
            "waf": "Operational Excellence",
            "service": "Azure OpenAI",
            "guid": "a3aec2c4-e243-46b0-936c-b45e17960eee",
            "id": "AOAI.14",
            "severity": "Medium",
            "link": "https://learn.microsoft.com/azure/ai-services/openai/how-to/monitoring"
        },
        {
            "category": "Operations Management",
            "subcategory": "Observability",
            "text": "If the diagnostics are not sufficient for you, consider using a gateway such as Azure API Managements in front of Azure OpenAI to log both incoming prompts and outgoing responses, where permitted",
            "waf": "Operational Excellence",
            "service": "Azure OpenAI",
            "guid": "fbdf4cc2-eec4-4d76-8c31-d25ffbb46a39",
            "id": "AOAI.15",
            "severity": "Low",
            "link": "https://techcommunity.microsoft.com/t5/apps-on-azure-blog/build-an-enterprise-ready-azure-openai-solution-with-azure-api/ba-p/3907562"
        },
        {
            "category": "Operations Management",
            "subcategory": "Infrastructure Deployment",
            "text": "Use Infrastructure as code to deploy the Azure OpenAI Service, model deployments, and all related resources",
            "waf": "Operational Excellence",
            "service": "Azure OpenAI",
            "guid": "3af30ed3-2947-498b-8178-a2c5a46ceb54",
            "id": "AOAI.16",
            "severity": "High",
            "link": "https://github.com/Azure-Samples/openai-enterprise-iac"
        },
        {
            "category": "Governance and Security",
            "subcategory": "Authentication",
            "text": "Use Microsoft Entra Authentication with Managed Identity instead of API Key",
            "waf": "Security",
            "service": "Azure OpenAI",
            "guid": "4350d092-d234-4292-a752-8537a551c5bf",
            "id": "AOAI.17",
            "severity": "High",
            "link": "https://learn.microsoft.com/azure/ai-services/openai/how-to/managed-identity"
        },
        {
            "category": "Responsible AI",
            "subcategory": "Evaluation",
            "text": "Evaluate the performance/accuracy of the system with a known golden dataset which has the inputs and the correct answers. Leverage capabilities in PromptFlow for Evaluation.",
            "waf": "Operational Excellence",
            "service": "Azure OpenAI",
            "guid": "4e4f1854-287d-45cd-a126-cc031af5b1fc",
            "id": "AOAI.18",
            "severity": "High",
            "link": "https://learn.microsoft.com/azure/machine-learning/prompt-flow/how-to-bulk-test-evaluate-flow?view=azureml-api-2"
        },
        {
            "category": "Operations Management",
            "subcategory": "Hosting model",
            "text": "Evaluate usage of Provisioned throughput model ",
            "waf": "Performance",
            "service": "Azure OpenAI",
            "guid": "68889535-e327-4897-b31b-67d67be5962a",
            "id": "AOAI.19",
            "severity": "High",
            "link": "https://learn.microsoft.com/azure/architecture/ai-ml/architecture/baseline-openai-e2e-chat#azure-openai---performance-efficiency"
        },
        {
            "category": "Responsible AI",
            "subcategory": "Content Safety",
            "text": "Review and implement Azure AI content safety",
            "waf": "Operational Excellence",
            "service": "Azure OpenAI",
            "guid": "cd288bed-6b17-4cb8-8454-51e1aed3453a",
            "id": "AOAI.2",
            "severity": "High",
            "link": "https://learn.microsoft.com/azure/ai-services/content-safety/overview"
        },
        {
            "category": "Operations Management",
            "subcategory": "Throughput definition",
            "text": "Define and evaluate the throughput of the system based on tokens & response per minute and align with requirements",
            "waf": "Performance",
            "service": "Azure OpenAI",
            "guid": "1193846d-697c-4b39-8ed1-6b2d186f0a02",
            "id": "AOAI.20",
            "severity": "High",
            "link": "https://learn.microsoft.com/azure/ai-services/openai/how-to/latency#system-level-throughput"
        },
        {
            "category": "Operations Management",
            "subcategory": "Latency improvement",
            "text": "Improve latency of the system by limiting token sizes, streaming options",
            "waf": "Performance",
            "service": "Azure OpenAI",
            "guid": "41addde6-8a47-47cd-bb48-61bc3bc10ae6",
            "id": "AOAI.21",
            "severity": "Medium",
            "link": "https://learn.microsoft.com/azure/ai-services/openai/how-to/latency#improve-performance"
        },
        {
            "category": "Operations Management",
            "subcategory": "Elasticity segregation",
            "text": "Estimate elasticity demands to determine synchronous and batch request segregation based on priority. For high priority, use synchronous approach and for low priority, asynchronous batch processing with queue is preferred",
            "waf": "Performance",
            "service": "Azure OpenAI",
            "guid": "6e25d4d5-a3ae-4c2c-9e24-36b0336cb45e",
            "id": "AOAI.22",
            "severity": "Medium",
            "link": "https://learn.microsoft.com/azure/ai-services/openai/how-to/latency#batching"
        },
        {
            "category": "Operations Management",
            "subcategory": "Benchmarking",
            "text": "Benchmark token consumption requirements based on estimated demands from consumers. Consider using the Azure OpenAI benchmarking tool to help you validate the throughput if you are using Provisioned Throughput Unit deployments",
            "waf": "Performance",
            "service": "Azure OpenAI",
            "guid": "5bda4332-4f24-4811-9331-82ba51752694",
            "id": "AOAI.23",
            "severity": "High",
            "link": "https://github.com/Azure/azure-openai-benchmark/"
        },
        {
            "category": "Operations Management",
            "subcategory": "Elasticity ",
            "text": "If you are using Provisioned Throughput Units (PTUs), consider deploying a token-per-minute (TPM) deployment for overflow requests. Use a gateway to route requests to the TPM deployment when the PTU limits are reached.",
            "waf": "Performance",
            "service": "Azure OpenAI",
            "guid": "4008ae7d-7e47-4432-96d8-bdcf55bce619",
            "id": "AOAI.24",
            "severity": "Medium",
            "link": "https://techcommunity.microsoft.com/t5/fasttrack-for-azure/optimizing-azure-openai-a-guide-to-limits-quotas-and-best/ba-p/4076268"
        },
        {
            "category": "Operations Management",
            "subcategory": "Model choice",
            "text": "Choose the right model for the right task. Pick models with right tradeoff between speed, quality of response and output complexity",
            "waf": "Performance",
            "service": "Azure OpenAI",
            "guid": "e8a13f98-8794-424d-9267-86d60b96c97b",
            "id": "AOAI.25",
            "severity": "High",
            "link": "https://learn.microsoft.com/azure/ai-services/openai/concepts/models"
        },
        {
            "category": "Operations Management",
            "subcategory": "Fine tuning",
            "text": "Have a baseline for performance without fine-tuning for knowing whether or not fine-tuning has improved model performance",
            "waf": "Performance",
            "service": "Azure OpenAI",
            "guid": "e9951904-8384-45c9-a6cb-2912156a1147",
            "id": "AOAI.26",
            "severity": "Medium",
            "link": "https://github.com/Azure/azure-openai-benchmark/"
        },
        {
            "category": "BC and DR",
            "subcategory": "Multi-region architecture",
            "text": "Deploy multiple OAI instances across regions",
            "waf": "Reliability",
            "service": "Azure OpenAI",
            "guid": "5e39f541-accc-4d97-a376-bcdb3750ab2a",
            "id": "AOAI.27",
            "severity": "Low",
            "link": "https://learn.microsoft.com/azure/architecture/ai-ml/architecture/baseline-openai-e2e-chat#azure-openai---reliability"
        },
        {
            "category": "BC and DR",
            "subcategory": "Load balancing",
            "text": "Implement retry & healthchecks with Gateway pattern like APIM",
            "waf": "Reliability",
            "service": "Azure OpenAI",
            "guid": "b039da6d-55d7-4c89-8adb-107d5325af62",
            "id": "AOAI.28",
            "severity": "High",
            "link": "https://learn.microsoft.com/azure/architecture/ai-ml/architecture/baseline-openai-e2e-chat#azure-openai---reliability"
        },
        {
            "category": "BC and DR",
            "subcategory": "Quotas",
            "text": "Ensure having adequate quotas of TPM & RPM for the workload",
            "waf": "Reliability",
            "service": "Azure OpenAI",
            "guid": "5ca44e46-85e2-4223-ace8-bb12308ca5f1",
            "id": "AOAI.29",
            "severity": "Medium",
            "link": "https://learn.microsoft.com/azure/ai-services/openai/how-to/quota?tabs=rest#introduction-to-quota"
        },
        {
            "category": "Responsible AI",
            "subcategory": "UX best practice",
            "text": "Review the considerations in HAI toolkit guidance and apply those interaction practices for the slution",
            "waf": "Operational Excellence",
            "service": "Azure OpenAI",
            "guid": "ec723923-7a15-42d6-ac5e-402925387e5c",
            "id": "AOAI.3",
            "severity": "Medium",
            "link": "https://www.microsoft.com/research/project/guidelines-for-human-ai-interaction/"
        },
        {
            "category": "BC and DR",
            "subcategory": "Load balancing",
            "text": "Deploy separate fine tuned models across regions if finetuning is employed",
            "waf": "Reliability",
            "service": "Azure OpenAI",
            "guid": "7f154e3a-a369-4282-ae7e-316183687a04",
            "id": "AOAI.30",
            "severity": "Medium",
            "link": "https://learn.microsoft.com/azure/ai-services/openai/how-to/business-continuity-disaster-recovery"
        },
        {
            "category": "BC and DR",
            "subcategory": "Data Backup and Disaster Recovery",
            "text": "Regularly backup and replicate critical data to ensure data availability and recoverability in case of data loss or system failures. Leverage Azure's backup and disaster recovery services to protect your data.",
            "waf": "Reliability",
            "service": "Azure OpenAI",
            "guid": "77a1f893-5bda-4433-84f2-4811633182ba",
            "id": "AOAI.31",
            "severity": "Medium",
            "link": "https://learn.microsoft.com/azure/backup/backup-overview"
        },
        {
            "category": "BC and DR",
            "subcategory": "SLA considerations",
            "text": "Azure AI search service tiers should be choosen to have a SLA ",
            "waf": "Reliability",
            "service": "Azure OpenAI",
            "guid": "95b96ad8-844c-4e3b-8b38-b876ba2cf204",
            "id": "AOAI.32",
            "severity": "High",
            "link": "https://www.microsoft.com/licensing/docs/view/Service-Level-Agreements-SLA-for-Online-Services?lang=1"
        },
        {
            "category": "Governance and Security",
            "subcategory": "Data Sensitivity",
            "text": "Classify data and sensitivity, labeling with Microsoft Purview before generating the embeddings and make sure to treat the embeddings generated with same sensitivity and classification",
            "waf": "Security",
            "service": "Azure OpenAI",
            "guid": "99013a5d-3ce4-474d-acbd-8682a6abca2a",
            "id": "AOAI.33",
            "severity": "Low",
            "link": "https://learn.microsoft.com/purview/purview"
        },
        {
            "category": "Governance and Security",
            "subcategory": "Encryption at Rest",
            "text": "Encrypt data used for RAG with SSE/Disk encryption with optional BYOK",
            "waf": "Security",
            "service": "Azure OpenAI",
            "guid": "4fda1dbf-3dd9-45d4-ac7c-891dca1f6d56",
            "id": "AOAI.34",
            "severity": "High",
            "link": "https://learn.microsoft.com/azure/ai-services/openai/how-to/use-your-data-securely"
        },
        {
            "category": "Governance and Security",
            "subcategory": "Transit Encryption",
            "text": "Ensure TLS is enforced for data in transit across data sources, AI search used for Retrieval-Augmented Generation (RAG) and LLM communication",
            "waf": "Security",
            "service": "Azure OpenAI",
            "guid": "59ae558b-937d-4498-9e11-12dbd7ba012f",
            "id": "AOAI.35",
            "severity": "High",
            "link": "https://learn.microsoft.com/azure/search/search-security-overview"
        },
        {
            "category": "Governance and Security",
            "subcategory": "Access Control",
            "text": "Use RBAC to manage access to Azure OpenAI services. Assign appropriate permissions to users and restrict access based on their roles and responsibilities",
            "waf": "Security",
            "service": "Azure OpenAI",
            "guid": "7b94ef6e-047d-42ea-8992-b1cd6e2054b2",
            "id": "AOAI.36",
            "severity": "High",
            "link": "https://learn.microsoft.com/azure/ai-services/openai/how-to/role-based-access-control"
        },
        {
            "category": "Governance and Security",
            "subcategory": "Data Masking and Redaction",
            "text": "Implement data encryption,  masking or redaction techniques to hide sensitive data or replace it with obfuscated values in non-production environments or when sharing data for testing or troubleshooting purposes",
            "waf": "Security",
            "service": "Azure OpenAI",
            "guid": "9769e4a6-91e8-4838-ac93-6667e13c0056",
            "id": "AOAI.37",
            "severity": "Medium",
            "link": "https://learn.microsoft.com/azure/security/fundamentals/data-encryption-best-practices"
        },
        {
            "category": "Governance and Security",
            "subcategory": "Threat Detection and Monitoring",
            "text": "Utilize Azure Defender to detect and respond to security threats and set up monitoring and alerting mechanisms to identify suspicious activities or breaches. Leverage Azure Sentinel for advanced threat detection and response",
            "waf": "Security",
            "service": "Azure OpenAI",
            "guid": "74b1e945-b459-4837-be7a-d6c6d3b375a5",
            "id": "AOAI.38",
            "severity": "High",
            "link": "https://learn.microsoft.com/azure/defender-for-cloud/defender-for-cloud-introduction"
        },
        {
            "category": "Governance and Security",
            "subcategory": "Data Retention and Disposal",
            "text": "Establish data retention and disposal policies to adhere to compliance regulations. Implement secure deletion methods for data that is no longer required and maintain an audit trail of data retention and disposal activities",
            "waf": "Security",
            "service": "Azure OpenAI",
            "guid": "c7acbe48-abe5-44cd-99f2-e87768468c55",
            "id": "AOAI.39",
            "severity": "Medium",
            "link": "https://techcommunity.microsoft.com/t5/azure-storage-blog/managing-long-term-log-retention-or-any-business-data/ba-p/2494791"
        },
        {
            "category": "Responsible AI",
            "subcategory": "Jail break Safety",
            "text": "Implement Prompt shields and groundedness detection using Content Safety ",
            "waf": "Operational Excellence",
            "service": "Azure OpenAI",
            "guid": "a9c27d9c-42bb-46bd-8c69-99a246f3389a",
            "id": "AOAI.4",
            "severity": "High",
            "link": "https://learn.microsoft.com/azure/ai-services/content-safety/concepts/jailbreak-detection"
        },
        {
            "category": "Governance and Security",
            "subcategory": "Data Privacy and Compliance",
            "text": "Ensure compliance with relevant data protection regulations, such as GDPR or HIPAA, by implementing privacy controls and obtaining necessary consents or permissions for data processing activities.",
            "waf": "Security",
            "service": "Azure OpenAI",
            "guid": "a775c6ee-95b9-46ad-a844-ce3b2b38b876",
            "id": "AOAI.40",
            "severity": "High",
            "link": "https://learn.microsoft.com/azure/compliance/"
        },
        {
            "category": "Governance and Security",
            "subcategory": "Employee Awareness and Training",
            "text": "Educate your employees about data security best practices, the importance of handling data securely, and potential risks associated with data breaches. Encourage them to follow data security protocols diligently.",
            "waf": "Security",
            "service": "Azure OpenAI",
            "guid": "ba2cf204-9901-43a5-b3ce-474dccbd8682",
            "id": "AOAI.41",
            "severity": "Medium"
        },
        {
            "category": "Governance and Security",
            "subcategory": "Environment segregation",
            "text": "Keep production data separate from development and testing data. Only use real sensitive data in production and utilize anonymized or synthetic data in development and test environments.",
            "waf": "Security",
            "service": "Azure OpenAI",
            "guid": "eae01e6e-842e-452f-9721-d928c1b1cd52",
            "id": "AOAI.42",
            "severity": "High"
        },
        {
            "category": "Governance and Security",
            "subcategory": "Index Segregation",
            "text": "If you have varying levels of data sensitivity, consider creating separate indexes for each level. For instance, you could have one index for general data and another for sensitive data, each governed by different access protocols",
            "waf": "Security",
            "service": "Azure OpenAI",
            "guid": "1e54a29a-9de3-499c-bd7b-28dc93555620",
            "id": "AOAI.43",
            "severity": "Medium"
        },
        {
            "category": "Governance and Security",
            "subcategory": "Sensitive Data in Separate Instances",
            "text": "Take segregation a step further by placing sensitive datasets in different instances of the service. Each instance can be controlled with its own specific set of RBAC policies",
            "waf": "Security",
            "service": "Azure OpenAI",
            "guid": "2bfe4564-b0d8-434a-948b-263e6dd60512",
            "id": "AOAI.44",
            "severity": "Medium"
        },
        {
            "category": "Governance and Security",
            "subcategory": "Embedding and Vector handling",
            "text": "Recognize that embeddings and vectors generated from sensitive information are themselves sensitive. This data should be afforded the same protective measures as the source material",
            "waf": "Security",
            "service": "Azure OpenAI",
            "guid": "a36498f6-dbad-438e-ad53-cc7ce1d7aaab",
            "id": "AOAI.45",
            "severity": "High"
        },
        {
            "category": "Governance and Security",
            "subcategory": "Access control",
            "text": "Apply RBAC to th data stores having embeddings and vectors and scope access based on role's access requirements",
            "waf": "Security",
            "service": "Azure OpenAI",
            "guid": "3571449a-b805-43d8-af89-dc7b33be2a1a",
            "id": "AOAI.46",
            "severity": "High",
            "link": "https://learn.microsoft.com/azure/ai-services/openai/how-to/role-based-access-control"
        },
        {
            "category": "Governance and Security",
            "subcategory": "Network security",
            "text": "Configure private endpoint for AI services to restrict service access within your network",
            "waf": "Security",
            "service": "Azure OpenAI",
            "guid": "27f7b9e9-1be1-4f38-aef3-9812bd463cbb",
            "id": "AOAI.47",
            "severity": "High",
            "link": "https://techcommunity.microsoft.com/t5/azure-architecture-blog/azure-openai-private-endpoints-connecting-across-vnet-s/ba-p/3913325"
        },
        {
            "category": "Governance and Security",
            "subcategory": "Network security",
            "text": "Enforce strict inbound and outbound traffic control with Azure Firewall and UDRs and limit the external integration points",
            "waf": "Security",
            "service": "Azure OpenAI",
            "guid": "ac8ac199-ebb9-41a3-9d90-cae2cc881370",
            "id": "AOAI.48",
            "severity": "High"
        },
        {
            "category": "Governance and Security",
            "subcategory": "Control Network Access",
            "text": "Implement network segmentation and access controls to restrict access to the LLM application only to authorized users and systems and prevent lateral movement",
            "waf": "Security",
            "service": "Azure OpenAI",
            "guid": "6f7c0cba-fe51-4464-add4-57e927138b82",
            "id": "AOAI.49",
            "severity": "High"
        },
        {
            "category": "Cost Optimization",
            "subcategory": "Token Optimization",
            "text": "Use prompt compression tools like LLMLingua or gprtrim",
            "waf": "Cost Optimization",
            "service": "Azure OpenAI",
            "guid": "7f42c78e-78cb-46a2-8ad1-90916e6a8d8f",
            "id": "AOAI.5",
            "severity": "Medium",
            "link": "https://www.microsoft.com/research/blog/llmlingua-innovating-llm-efficiency-with-prompt-compression/"
        },
        {
            "category": "Governance and Security",
            "subcategory": "Secure APIs and Endpoints",
            "text": "Ensure that APIs and endpoints used by the LLM application are properly secured with authentication and authorization mechanisms, such as Managed identities,  API keys or OAuth, to prevent unauthorized access.",
            "waf": "Security",
            "service": "Azure OpenAI",
            "guid": "1102cac6-eae0-41e6-b842-e52f4721d928",
            "id": "AOAI.50",
            "severity": "High",
            "link": "https://learn.microsoft.com/azure/ai-services/openai/how-to/managed-identity"
        },
        {
            "category": "Governance and Security",
            "subcategory": "Implement Strong Authentication",
            "text": "Enforce strong end user authentication mechanisms, such as multi-factor authentication, to prevent unauthorized access to the LLM application and associated network resources",
            "waf": "Security",
            "service": "Azure OpenAI",
            "guid": "c1b1cd52-1e54-4a29-a9de-399cfd7b28dc",
            "id": "AOAI.51",
            "severity": "Medium",
            "link": "https://techcommunity.microsoft.com/t5/azure-architecture-blog/security-best-practices-for-genai-applications-openai-in-azure/ba-p/4027885"
        },
        {
            "category": "Governance and Security",
            "subcategory": "Use Network Monitoring",
            "text": "Implement network monitoring tools to detect and analyze network traffic for any suspicious or malicious activities. Enable logging to capture network events and facilitate forensic analysis in case of security incidents",
            "waf": "Security",
            "service": "Azure OpenAI",
            "guid": "93555620-2bfe-4456-9b0d-834a348b263e",
            "id": "AOAI.52",
            "severity": "Medium"
        },
        {
            "category": "Governance and Security",
            "subcategory": "Security Audits and Penetration Testing",
            "text": "Conduct security audits and penetration testing to identify and address any network security weaknesses or vulnerabilities in the LLM application's network infrastructure",
            "waf": "Security",
            "service": "Azure OpenAI",
            "guid": "6dd60512-a364-498f-9dba-d38ead53cc7c",
            "id": "AOAI.53",
            "severity": "Medium"
        },
        {
            "category": "Governance and Security",
            "subcategory": "Infrastructure Deployment",
            "text": "Azure AI Services are properly tagged for better management",
            "waf": "Operational Excellence",
            "guid": "e1d7aaab-3571-4449-ab80-53d89f89dc7b",
            "id": "AOAI.54",
            "service": "Azure OpenAI",
            "severity": "Low",
            "link": "https://learn.microsoft.com/azure/azure-resource-manager/management/tag-resources?tabs=json"
        },
        {
            "category": "Governance and Security",
            "subcategory": "Infrastructure Deployment",
            "text": "Azure AI Service accounts follows organizational naming conventions",
            "waf": "Operational Excellence",
            "service": "Azure OpenAI",
            "guid": "77036e5e-6b4b-4ed3-b503-547c1347dc56",
            "id": "AOAI.55",
            "severity": "Low",
            "link": "https://learn.microsoft.com/azure/cloud-adoption-framework/ready/azure-best-practices/resource-abbreviations"
        },
        {
            "category": "Governance and Security",
            "subcategory": "Diagnostics Logging",
            "text": "Diagnostic logs in Azure AI services resources should be enabled",
            "waf": "Operational Excellence",
            "service": "Azure OpenAI",
            "guid": "028a71ff-e1ce-415d-b3f0-d5e772d41e36",
            "id": "AOAI.56",
            "severity": "High",
            "link": "https://learn.microsoft.com/azure/ai-services/diagnostic-logging"
        },
        {
            "category": "Identity and Access Management",
            "subcategory": "Entra ID based access",
            "text": "Key access (local authentication) is recommended to be disabled for security.  After disabling key based access, Microsoft Entra ID becomes the only access method, which allows maintaining minimum privilege principle and granular control. ",
            "waf": "Security",
            "service": "Azure OpenAI",
            "guid": "11cc57b4-a4b1-4410-b439-58a8c2289b3d",
            "id": "AOAI.57",
            "severity": "High",
            "link": "https://learn.microsoft.com/azure/ai-services/authentication"
        },
        {
            "category": "Governance and Security",
            "subcategory": "Secure Key Management",
            "text": "Store and manage keys securely using Azure Key Vault. Avoid hard-coding or embedding sensitive keys within your LLM application's code and retrieve them securely from Azure Key Vault using managed identities",
            "waf": "Security",
            "service": "Azure OpenAI",
            "guid": "6b57cfc6-5546-41e1-a3e3-453a3c863964",
            "id": "AOAI.58",
            "severity": "High",
            "link": "https://learn.microsoft.com/azure/key-vault/general/best-practices"
        },
        {
            "category": "Governance and Security",
            "subcategory": "Key Rotation and Expiration",
            "text": "Regularly rotate and expire keys stored in Azure Key Vault to minimize the risk of unauthorized access.",
            "waf": "Security",
            "service": "Azure OpenAI",
            "guid": "8b652d6c-15f5-4129-9539-8e6ded227dd1",
            "id": "AOAI.59",
            "severity": "High",
            "link": "https://learn.microsoft.com/azure/key-vault/general/best-practices"
        },
        {
            "category": "Cost Optimization",
            "subcategory": "Token Optimization",
            "text": "Use tiktoken to understand token sizes for token optimizations in conversational mode",
            "waf": "Cost Optimization",
            "service": "Azure OpenAI",
            "guid": "adfe27be-e297-401a-a352-baaab79b088d",
            "id": "AOAI.6",
            "severity": "High",
            "link": "https://github.com/openai/tiktoken"
        },
        {
            "category": "Governance and Security",
            "subcategory": "Secure coding practice",
            "text": "Follow secure coding practices to prevent common vulnerabilities such as injection attacks, cross-site scripting (XSS), or security misconfigurations",
            "waf": "Security",
            "service": "Azure OpenAI",
            "guid": "42b06c21-d799-49a6-96f4-389a7f42c78e",
            "id": "AOAI.60",
            "severity": "High",
            "link": "https://learn.microsoft.com/azure/security/develop/secure-dev-overview"
        },
        {
            "category": "Governance and Security",
            "subcategory": "Patching and updates",
            "text": "Setup a process to regularly update and patch the LLM libraries and other system components",
            "waf": "Security",
            "service": "Azure OpenAI",
            "guid": "78c06a73-a22a-4495-9e6a-8dc4a20e27c3",
            "id": "AOAI.61",
            "severity": "High",
            "link": "https://learn.microsoft.com/azure/devops/repos/security/github-advanced-security-dependency-scanning?view=azure-devops"
        },
        {
            "category": "Responsible AI",
            "subcategory": "Governance",
            "text": "Adhere to Azure OpenAI or other LLMs terms of use, policies and guidance and allowed use cases",
            "waf": "Operational Excellence",
            "service": "Azure OpenAI",
            "guid": "e29711b1-352b-4eee-879b-588defc4972c",
            "id": "AOAI.62",
            "severity": "High",
            "link": "https://learn.microsoft.com/legal/cognitive-services/openai/code-of-conduct"
        },
        {
            "category": "Cost Optimization",
            "subcategory": "Cost familiarization",
            "text": "Understand difference in cost of base models and fine tuned models and token step sizes",
            "waf": "Cost Optimization",
            "service": "Azure OpenAI",
            "guid": "d3cd21bf-7703-46e5-b6b4-bed3d503547c",
            "id": "AOAI.63",
            "severity": "Medium",
            "link": "https://learn.microsoft.com/azure/ai-services/openai/how-to/manage-costs#base-series-and-codex-series-fine-tuned-models"
        },
        {
            "category": "Cost Optimization",
            "subcategory": "Batch processing",
            "text": "Batch requests, where possible, to minimize the per-call overhead which can reduce overall costs. Ensure you optimize batch size",
            "waf": "Cost Optimization",
            "service": "Azure OpenAI",
            "guid": "1347dc56-028a-471f-be1c-e15dd3f0d5e7",
            "id": "AOAI.64",
            "severity": "High",
            "link": "https://learn.microsoft.com/azure/ai-services/openai/how-to/latency#batching"
        },
        {
            "category": "Cost Optimization",
            "subcategory": "Cost monitoring",
            "text": "Set up a cost tracking system that monitors model usage and use that information to help inform model choices and prompt sizes",
            "waf": "Cost Optimization",
            "service": "Azure OpenAI",
            "guid": "72d41e36-11cc-457b-9a4b-1410d43958a8",
            "id": "AOAI.65",
            "severity": "Medium",
            "link": "https://learn.microsoft.com/azure/ai-services/openai/how-to/manage-costs"
        },
        {
            "category": "Cost Optimization",
            "subcategory": "Token limit",
            "text": "Set a maximum limit on the number of tokens per model response. Optimize the size to ensure it is large enough for a valid response",
            "waf": "Cost Optimization",
            "service": "Azure OpenAI",
            "guid": "166cd072-af9b-4141-a898-a535e737897e",
            "id": "AOAI.66",
            "severity": "Medium",
            "link": "https://learn.microsoft.com/azure/ai-services/openai/how-to/quota?tabs=rest#understanding-rate-limits"
        },
        {
            "category": "Operations Management",
            "subcategory": "AI Search Reliability",
            "text": "Review the guidance provided on setting up AI search for Reliability",
            "waf": "Operational Excellence",
            "service": "Azure OpenAI",
            "guid": "71ca7da8-cfa9-462a-8594-946da97dc3a2",
            "id": "AOAI.67",
            "severity": "Medium",
            "link": "https://learn.microsoft.com/azure/search/search-reliability"
        },
        {
            "category": "Operations Management",
            "subcategory": "AI Search Vector Limits",
            "text": "Plan and manage AI Search Vector storage",
            "waf": "Operational Excellence",
            "service": "Azure OpenAI",
            "guid": "3266b225-86f4-4a16-92bd-ddea8a487cde",
            "id": "AOAI.68",
            "severity": "Medium",
            "link": "https://learn.microsoft.com/azure/search/vector-search-index-size?tabs=portal-vector-quota"
        },
        {
            "category": "Operations Management",
            "subcategory": "DevOps",
            "text": "Apply LLMOps practices to automate the lifecycle management of your GenAI applications",
            "waf": "Operational Excellence",
            "service": "Azure OpenAI",
            "guid": "b4861bc3-bc14-4aeb-9e66-e8d9a3aec218",
            "id": "AOAI.69",
            "severity": "Medium",
            "link": "https://learn.microsoft.com/azure/machine-learning/prompt-flow/how-to-end-to-end-llmops-with-prompt-flow?view=azureml-api-2"
        },
        {
            "category": "Cost Optimization",
            "subcategory": "Costing Model",
            "text": "Evaluate usage of billing models - PAYG vs PTU",
            "waf": "Cost Optimization",
            "service": "Azure OpenAI",
            "guid": "aa80932c-8ec9-4d1b-a770-26e5e6beba9e",
            "id": "AOAI.7",
            "severity": "High",
            "link": "https://learn.microsoft.com/azure/ai-services/openai/how-to/provisioned-throughput-onboarding#understanding-the-provisioned-throughput-purchase-model"
        },
        {
            "category": "Operations Management",
            "subcategory": "DevOps",
            "text": "Evaluate the quality of  prompts and applications when switching between model versions",
            "waf": "Operational Excellence",
            "service": "Azure OpenAI",
            "guid": "e6436b07-36db-455f-9796-03334bdf9cc2",
            "id": "AOAI.70",
            "severity": "Medium",
            "link": "https://techcommunity.microsoft.com/t5/ai-azure-ai-services-blog/how-to-control-azure-openai-models/ba-p/4146793"
        },
        {
            "category": "Operations Management",
            "subcategory": "Development",
            "text": "Evaluate, monitor and refine your GenAI apps for features like groundedness, relevance, accuracy, coherence, fluency, \ufffd",
            "waf": "Operational Excellence",
            "service": "Azure OpenAI",
            "guid": "3418db61-2712-4650-9bb4-7a393a080327",
            "id": "AOAI.71",
            "severity": "Medium",
            "link": "https://learn.microsoft.com/azure/machine-learning/prompt-flow/concept-model-monitoring-generative-ai-evaluation-metrics?view=azureml-api-2"
        },
        {
            "category": "Operations Management",
            "subcategory": "Development",
            "text": "Evaluate your Azure AI Search results based on different search parameters",
            "waf": "Operational Excellence",
            "service": "Azure OpenAI",
            "guid": "294798b1-578b-4219-a46c-eb5443513592",
            "id": "AOAI.72",
            "severity": "Medium"
        },
        {
            "category": "Operations Management",
            "subcategory": "Development",
            "text": "Look at fine tuning models as way of increasing accuracy only when you have tried other basic approaches like prompt engineering and RAG with your data",
            "waf": "Operational Excellence",
            "service": "Azure OpenAI",
            "guid": "2744293b-b628-4537-a551-19b08e8f5854",
            "id": "AOAI.73",
            "severity": "Medium",
            "link": "https://learn.microsoft.com/azure/ai-services/openai/concepts/fine-tuning-considerations"
        },
        {
            "category": "Operations Management",
            "subcategory": "Development",
            "text": "Use prompt engineering techniques to improve the accuracy of LLM responses",
            "waf": "Operational Excellence",
            "service": "Azure OpenAI",
            "guid": "287d9cec-166c-4d07-8af9-b141a898a535",
            "id": "AOAI.74",
            "severity": "Medium",
            "link": "https://learn.microsoft.com/azure/ai-services/openai/concepts/advanced-prompt-engineering?pivots=programming-language-chat-completions"
        },
        {
            "category": "Governance and Security",
            "subcategory": "Security Audits and Penetration Testing",
            "text": "Red team your GenAI applications",
            "waf": "Security",
            "service": "Azure OpenAI",
            "guid": "e737897e-71ca-47da-acfa-962a1594946d",
            "id": "AOAI.75",
            "severity": "Medium",
            "link": "https://learn.microsoft.com/azure/ai-services/openai/concepts/red-teaming"
        },
        {
            "category": "Operations Management",
            "subcategory": "End user feedback",
            "text": "Provide end users with scoring options for LLM responses and track these scores. ",
            "waf": "Operational Excellence",
            "service": "Azure OpenAI",
            "guid": "edb117e6-76aa-4f66-aca4-8e5a95f2223e",
            "id": "AOAI.76",
            "severity": "Medium",
            "link": "https://www.microsoft.com/haxtoolkit/guideline/encourage-granular-feedback/"
        },
        {
            "category": "Cost Optimization",
            "subcategory": "Quota Management",
            "text": "Consider Quota management practices",
            "waf": "Cost Optimization",
            "service": "Azure OpenAI",
            "guid": "d5f3547c-c346-4d81-9028-a71ffe1b9b5d",
            "id": "AOAI.8",
            "severity": "High",
            "link": "https://techcommunity.microsoft.com/t5/fasttrack-for-azure/optimizing-azure-openai-a-guide-to-limits-quotas-and-best/ba-p/4076268"
        },
        {
            "category": "Operations Management",
            "subcategory": "Load Balancing",
            "text": "Use Load balancer solutions like APIM based gateway for balancing load and capacity across services and regions",
            "waf": "Operational Excellence",
            "service": "Azure OpenAI",
            "guid": "9de0d5d7-31d4-41e3-911c-817bfafbc410",
            "id": "AOAI.9",
            "severity": "Medium",
            "link": "https://github.com/Azure/aoai-apim/blob/main/README.md"
        }
    ],
    "categories": [
        {
            "name": "Identity and Access Management"
        },
        {
            "name": "Network Topology and Connectivity"
        },
        {
            "name": "BC and DR"
        },
        {
            "name": "Governance and Security"
        },
        {
            "name": "Cost Governance"
        },
        {
            "name": "Operations Management"
        },
        {
            "name": "Application Deployment"
        },
        {
            "name": "Responsible AI"
        }
    ],
    "waf": [
        {
            "name": "Reliability"
        },
        {
            "name": "Security"
        },
        {
            "name": "Cost"
        },
        {
            "name": "Operations"
        },
        {
            "name": "Performance"
        }
    ],
    "yesno": [
        {
            "name": "Yes"
        },
        {
            "name": "No"
        }
    ],
    "status": [
        {
            "name": "Not verified",
            "description": "This check has not been looked at yet"
        },
        {
            "name": "Open",
            "description": "There is an action item associated to this check"
        },
        {
            "name": "Fulfilled",
            "description": "This check has been verified, and there are no further action items associated to it"
        },
        {
            "name": "Not required",
            "description": "Recommendation understood, but not needed by current requirements"
        },
        {
            "name": "N/A",
            "description": "Not applicable for current design"
        }
    ],
    "severities": [
        {
            "name": "High"
        },
        {
            "name": "Medium"
        },
        {
            "name": "Low"
        }
    ],
    "metadata": {
        "name": "Azure OpenAI Review",
        "state": "Preview",
        "waf": "all",
        "timestamp": "July 24, 2024"
    }
}