Introduces MSGraph sqlite db. Closes #304 (#324)

* Introduces MSGraph sqlite db. Closes #304

* Moves comment to the right place

Author: waldekmastykarz

m365-developer-proxy-abstractions/ILogger.cs

@@ -36,7 +36,7 @@ public enum LogLevel {
     Error
 }
 
-public interface ILogger {
+public interface ILogger: ICloneable {
     public LogLevel LogLevel { get; set; }
 
     public void LogRequest(string[] message, MessageType messageType, LoggingContext? context = null);

m365-developer-proxy-abstractions/ProxyUtils.cs

@@ -2,6 +2,7 @@
 // Licensed under the MIT License.
 
 using System.Text.RegularExpressions;
+using Microsoft.Data.Sqlite;
 using Titanium.Web.Proxy.Http;
 using Titanium.Web.Proxy.Models;
 
@@ -22,6 +23,23 @@ public static class ProxyUtils {
     private static readonly Regex deprecationRegex = new Regex("^[a-z]+_v2$", RegexOptions.IgnoreCase);
     private static readonly Regex functionCallRegex = new Regex(@"^[a-z]+\(.*\)$", RegexOptions.IgnoreCase);
 
+    // v1 refers to v1 of the db schema, not the graph version
+    public static string MsGraphDbFilePath => Path.Combine(AppFolder!, "msgraph-openapi-v1.db");
+    private static SqliteConnection? _msGraphDbConnection;
+    public static SqliteConnection MsGraphDbConnection {
+        get {
+            if (_msGraphDbConnection is null) {
+                _msGraphDbConnection = new SqliteConnection($"Data Source={MsGraphDbFilePath}");
+                _msGraphDbConnection.Open();
+            }
+
+            return _msGraphDbConnection;
+        }
+    }
+
+    // doesn't end with a path separator
+    public static string? AppFolder => Path.GetDirectoryName(AppContext.BaseDirectory);
+
     public static bool IsGraphRequest(Request request) => IsGraphUrl(request.RequestUri);
 
     public static bool IsGraphUrl(Uri uri) => 
@@ -76,9 +94,7 @@ public static string ReplacePathTokens(string? path) {
             return path ?? string.Empty;
         }
 
-        // doesn't end with a path separator
-        var appFolder = Path.GetDirectoryName(AppContext.BaseDirectory);
-        return path.Replace("~appFolder", appFolder, StringComparison.OrdinalIgnoreCase);
+        return path.Replace("~appFolder", AppFolder, StringComparison.OrdinalIgnoreCase);
     }
 
     // from: https://github.com/microsoftgraph/microsoft-graph-explorer-v4/blob/db86b903f36ef1b882996d46aee52cd49ed4444b/src/app/utils/query-url-sanitization.ts

m365-developer-proxy-abstractions/m365-developer-proxy-abstractions.csproj

@@ -9,6 +9,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="7.0.10" />
     <PackageReference Include="Microsoft.Extensions.Configuration" Version="7.0.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="7.0.4" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="7.0.0" />

m365-developer-proxy-plugins/Guidance/GraphSelectGuidancePlugin.cs

@@ -1,10 +1,7 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-using System.Text.RegularExpressions;
 using Microsoft.Extensions.Configuration;
-using Microsoft.OpenApi.Models;
-using Microsoft.OpenApi.Readers;
 using Microsoft365.DeveloperProxy.Abstractions;
 using Titanium.Web.Proxy.Http;
 
@@ -13,94 +10,17 @@ namespace Microsoft365.DeveloperProxy.Plugins.Guidance;
 public class GraphSelectGuidancePlugin : BaseProxyPlugin
 {
     public override string Name => nameof(GraphSelectGuidancePlugin);
-    private readonly Dictionary<string, OpenApiDocument> _openApiDocuments = new();
 
     public override void Register(IPluginEvents pluginEvents,
                             IProxyContext context,
                             ISet<UrlToWatch> urlsToWatch,
                             IConfigurationSection? configSection = null)
     {
-        var proxyFolder = Path.GetDirectoryName(context.Configuration.ConfigFile);
-        var stopwatch = new System.Diagnostics.Stopwatch();
-        stopwatch.Start();
-        LoadOpenAPIFiles(proxyFolder!);
-        stopwatch.Stop();
-        UpdateOpenAPIGraphFilesIfNecessary(proxyFolder!);
-
         base.Register(pluginEvents, context, urlsToWatch, configSection);
 
         pluginEvents.AfterResponse += AfterResponse;
     }
 
-    private async Task UpdateOpenAPIGraphFilesIfNecessary(string proxyFolder)
-    {
-        _logger?.LogDebug("Checking for updated OpenAPI files...");
-
-        var modified = false;
-        var versions = new[] { "v1.0", "beta" };
-        foreach (var version in versions)
-        {
-            try
-            {
-                var file = new FileInfo(Path.Combine(proxyFolder, "plugins", $"graph-{version.Replace(".", "_")}-openapi.yaml"));
-                _logger?.LogDebug($"Checking for updated OpenAPI file for {file}...");
-                if (file.LastWriteTime.Date == DateTime.Now.Date)
-                {
-                    _logger?.LogDebug($"File {file} already updated today");
-                    // file already updated today
-                    continue;
-                }
-
-                var url = $"https://raw.githubusercontent.com/microsoftgraph/msgraph-metadata/master/openapi/{version}/openapi.yaml";
-                _logger?.LogDebug($"Downloading OpenAPI file from {url}...");
-
-                var client = new HttpClient();
-                var response = await client.GetStringAsync(url);
-                File.WriteAllText(file.FullName, response);
-
-                _logger?.LogDebug($"Downloaded OpenAPI file from {url} to {file}");
-                modified = true;
-            }
-            catch (Exception ex)
-            {
-                _logger?.LogError(ex.Message);
-            }
-        }
-
-        if (modified)
-        {
-            LoadOpenAPIFiles(proxyFolder);
-        }
-    }
-
-    private async void LoadOpenAPIFiles(string proxyFolder)
-    {
-        _logger?.LogDebug("Loading OpenAPI files...");
-
-        var versions = new[] { "v1.0", "beta" };
-        foreach (var version in versions)
-        {
-            var file = new FileInfo(Path.Combine(proxyFolder, "plugins", $"graph-{version.Replace(".", "_")}-openapi.yaml"));
-            _logger?.LogDebug($"Loading OpenAPI file for {file}...");
-
-            if (!file.Exists)
-            {
-                _logger?.LogDebug($"File {file} does not exist");
-                continue;
-            }
-
-            try {
-                var openApiDocument = await new OpenApiStreamReader().ReadAsync(file.OpenRead());
-                _openApiDocuments[version] = openApiDocument.OpenApiDocument;
-                
-                _logger?.LogDebug($"Added OpenAPI file {file} for {version}");
-            }
-            catch (Exception ex) {
-                _logger?.LogDebug($"Error loading OpenAPI file {file}: {ex.Message}");
-            }
-        }
-    }
-
     private async Task AfterResponse(object? sender, ProxyResponseArgs e)
     {
         Request request = e.Session.HttpClient.Request;
@@ -134,25 +54,23 @@ private bool EndpointSupportsSelect(string graphVersion, string relativeUrl)
     {
         var fallback = relativeUrl.Contains("$value", StringComparison.OrdinalIgnoreCase);
 
-        if (!_openApiDocuments.ContainsKey(graphVersion))
+        try
         {
-            return fallback;
+            var dbConnection = ProxyUtils.MsGraphDbConnection;
+            // lookup information from the database
+            var selectEndpoint = dbConnection.CreateCommand();
+            selectEndpoint.CommandText = "SELECT hasSelect FROM endpoints WHERE path = @path AND graphVersion = @graphVersion";
+            selectEndpoint.Parameters.AddWithValue("@path", relativeUrl);
+            selectEndpoint.Parameters.AddWithValue("@graphVersion", graphVersion);
+            var result = selectEndpoint.ExecuteScalar();
+            var hasSelect = result != null && Convert.ToInt32(result) == 1;
+            return hasSelect;
         }
-
-        var relativeUrlPattern = Regex.Replace(relativeUrl, @"{[^}]+}", @"{[a-zA-Z-]+}");
-        var relativeUrlRegex = new Regex($"^{relativeUrlPattern}$");
-
-        var openApiDocument = _openApiDocuments[graphVersion];
-        var pathString = openApiDocument.Paths.Keys.FirstOrDefault(k => relativeUrlRegex.IsMatch(k));
-        if (pathString is null ||
-            !openApiDocument.Paths[pathString].Operations.ContainsKey(OperationType.Get))
+        catch (Exception ex)
         {
+            _logger?.LogError(ex.Message);
             return fallback;
         }
-
-        var operation = openApiDocument.Paths[pathString].Operations[OperationType.Get];
-        var parameters = operation.Parameters;
-        return parameters.Any(p => p.Name == "$select");
     }
 
     private static string GetSelectParameterGuidanceUrl() => "https://aka.ms/m365/proxy/guidance/select";