merge

Author: 11zhouxuan

poetry.lock

@@ -22,7 +22,7 @@ python = "^3.9"
 typer = {extras = ["all"], version = "^0.13.0"}
 rich = "^13.9.4"
 boto3 = "^1.35.0"
-questionary = "^2.0.1"
+questionary = "^2.1.0"
 requests = "^2.26"
 pydantic = "^2.9.2"
 sagemaker = "^2.237.0"

pyproject.toml

@@ -18,8 +18,10 @@ Parameters:
     Description: The listener to be used for the ECS Endpoint
   PublicLoadBalancerSecurityGroup:
     Type: String
-    Default: "sg-0d059e1ba522921fe"
     Description: The security group to be used for the ECS Endpoint
+  APIRouterSecurityGroup:
+    Type: String
+    Description: The security group to be used for the API Router
   LambdaDeploymentHelperArn:
     Type: String
     Description: The ARN of the Lambda function for capacity provider association
@@ -35,23 +37,19 @@ Parameters:
     Description: The instance type to be used for the ECS Endpoint
   ModelId:
     Type: String
-    Default: "qwen2-5-7b"
     Description: The emd model ID to be used for the ECS Endpoint
   ModelTag:
     Type: String
     Default: "dev"
     Description: The model tag to be used for the ECS Endpoint
   FrameWorkType:
     Type: String
-    Default: "awq"
     Description: The framework type to be used for the ECS Endpoint
   ServiceType:
     Type: String
-    Default: "instruct"
     Description: The service type to be used for the ECS Endpoint
   EngineType:
     Type: String
-    Default: "qwen2-5-72b"
     Description: The engine type to be used for the ECS Endpoint
   DesiredCapacity:
     Type: Number
@@ -171,6 +169,12 @@ Resources:
     Type: Custom::GetPriorityNumber
     Properties:
       ServiceToken: !Ref LambdaDeploymentHelperArn
+  ConvertDnsName:
+    Type: Custom::ConvertDnsName
+    Properties:
+      ServiceToken: !Ref LambdaDeploymentHelperArn
+      ModelName: !Join ['', [!Ref ModelId, '/', !Ref ModelTag]]
+
   ContainerHostSecurityGroup:
     Type: AWS::EC2::SecurityGroup
     Properties:
@@ -249,6 +253,8 @@ Resources:
           PortMappings:
             - ContainerPort: !Ref ContainerPort
               HostPort: !Ref ContainerPort
+              Name: model-port
+              AppProtocol: http
             - ContainerPort: 80
               HostPort: 80
           # Command: [--gpus, all]
@@ -261,11 +267,11 @@ Resources:
           LinuxParameters:
             sharedMemorySize: 1024
           healthCheck:
-            command: ["CMD-SHELL","curl -f http://localhost:8080/health || exit 1"]
-            interval: 120
-            retries: 10
-            timeout: 60
-            startPeriod: 120
+            command: ["CMD-SHELL","echo 'Server is running' || exit 0"]
+            interval: 30
+            retries: 3
+            timeout: 5
+            startPeriod: 10
           LogConfiguration:
             LogDriver: awslogs
             Options:
@@ -310,11 +316,31 @@ Resources:
         - ContainerName: !Sub '${AWS::StackName}'
           ContainerPort: !Ref ContainerPort
           TargetGroupArn: !Ref ServiceTargetGroup
+      ServiceConnectConfiguration:
+        Enabled: true
+        Namespace: emd-service-connect-namespace
+        Services:
+          - PortName: model-port
+            DiscoveryName: !GetAtt ConvertDnsName.DnsName
+            ClientAliases:
+              - Port: !Ref ContainerPort
+                DnsName: !GetAtt ConvertDnsName.DnsName
   ServiceSecurityGroup:
     Type: 'AWS::EC2::SecurityGroup'
     Properties:
       GroupDescription: Security group for service
       VpcId: !Ref VPCID
+
+  ServiceSecurityGroupIngress:
+    Type: AWS::EC2::SecurityGroupIngress
+    Properties:
+      GroupId: !GetAtt ServiceSecurityGroup.GroupId
+      IpProtocol: tcp
+      FromPort: !Ref ContainerPort
+      ToPort: !Ref ContainerPort
+      SourceSecurityGroupId: !Ref APIRouterSecurityGroup
+      Description: Allow traffic from API router service to model service
+
   ServiceTargetGroup:
     Type: 'AWS::ElasticLoadBalancingV2::TargetGroup'
     Properties:
@@ -360,10 +386,19 @@ Resources:
         - Type: forward
           TargetGroupArn: !Ref ServiceTargetGroup
 
+  ForceApiRouterDeployment:
+    Type: Custom::ForceApiRouterDeployment
+    DependsOn: Service
+    Properties:
+      ServiceToken: !Ref LambdaDeploymentHelperArn
+
 Outputs:
   Model:
-    Description: Model ID used to generate the response
+    Description: Model ID used to generate the response.
     Value: !Join ['', [!Ref ModelId, '/', !Ref ModelTag]]
-  PublicLoadBalancerDNSName:
-    Description: The DNS name of the public load balancer. To use HTTPS, create an SSL certificate in AWS Certificate Manager and attach it to the load balancer.
-    Value: !Join ['', ['http://', !Ref DNSName, '/', !Ref ModelId, '/', !Ref ModelTag]]
+  BaseURL:
+    Description: Please use this URL for the OpenAI-compatible model API, like /v1/chat/completions. To use HTTPS, create an SSL certificate in AWS Certificate Manager and attach it to the load balancer.
+    Value: !Join ['', ['http://', !Ref DNSName]]
+  ECSServiceConnect:
+    Description: Service Connect allows for service-to-service communications with automatic discovery using short names and standard ports.
+    Value: !Join ['', ['http://', !GetAtt ConvertDnsName.DnsName, ':', !Ref ContainerPort]]

src/emd/cfn/ecs/template.yaml

@@ -42,6 +42,9 @@ Parameters:
     Type: String
     Description: The name of the SageMaker Endpoint
     Default: "Auto-generate"
+  DNSName:
+    Type: String
+    Description: The DNS name of the public load balancer. To use HTTPS, create an SSL certificate in AWS Certificate Manager and attach it to the load balancer.
 
 Conditions:
   UseDefaultEndpointName: !Equals [!Ref SageMakerEndpointName, "Auto-generate"]
@@ -153,8 +156,11 @@ Resources:
 
 Outputs:
   Model:
-    Description: Model ID used to generate the response
+    Description: Model ID used to generate the response.
     Value: !Join ['', [!Ref ModelId, '/', !Ref ModelTag]]
+  BaseURL:
+    Description: Please use this URL for the OpenAI-compatible model API, like /v1/chat/completions. To use HTTPS, create an SSL certificate in AWS Certificate Manager and attach it to the load balancer.
+    Value: !Join ['', ['http://', !Ref DNSName]]
   SageMakerEndpointName:
-    Description: The name of the SageMaker Endpoint
+    Description: You can use invoke the endpoint by using the SageMaker runtime API.
     Value: !GetAtt SageMakerEndpoint.EndpointName

src/emd/cfn/sagemaker_realtime/template.yaml

@@ -170,7 +170,7 @@ def deploy_vpc_template(region):
     return vpc_id, subnets
 
 
-def deploy_ecs_cluster_template(region, vpc_id, subnets, api_router_uri, use_spot):
+def deploy_ecs_cluster_template(region, vpc_id, subnets, use_spot):
     client = boto3.client("cloudformation", region_name=region)
     stack_name = "EMD-ECS-Cluster"
     template_path = f"{CFN_ROOT_PATH}/shared/ecs_cluster.yaml"
@@ -187,10 +187,6 @@ def deploy_ecs_cluster_template(region, vpc_id, subnets, api_router_uri, use_spo
                 "ParameterKey": "Subnets",
                 "ParameterValue": subnets,
             },
-            {
-                "ParameterKey": "APIRouterImageURI",
-                "ParameterValue": api_router_uri,
-            },
             {
                 "ParameterKey": "UseSpot",
                 "ParameterValue": "yes" if use_spot else "no",
@@ -222,10 +218,10 @@ def deploy_ecs_cluster(region, vpc_id=None, subnets=None, use_spot=False):
     update_parameters_file("parameters.json", {"VPCID": vpc_id, "Subnets": subnets})
 
     # Build and push Fargate image to ECR as the OpenAI compatible API router
-    api_router_uri = build_router_image(region)
+    # api_router_uri = build_router_image(region)
 
     # Deploy the ECS cluster
-    deploy_ecs_cluster_template(region, vpc_id, subnets, api_router_uri, use_spot)
+    deploy_ecs_cluster_template(region, vpc_id, subnets, use_spot)
 
 if __name__ == "__main__":
     deploy_ecs_cluster("us-east-1")