updated to TorchSharp 0.103.0

Author: lostmsu

app/MinGPTProgram.cs

@@ -1,16 +1,11 @@
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
+using System.Diagnostics;
 using System.IO;
-using System.Linq;
 using System.Text;
 
 using LostTech.Torch.NN;
 
 using ShellProgressBar;
 
-using TorchSharp;
-
 using static TorchSharp.torch;
 using static TorchSharp.torch.nn;
 
@@ -29,7 +24,7 @@
     }
 });
 
-byte[] itob = vocab.ToArray();
+byte[] itob = [.. vocab];
 var btoi = Enumerable.Range(0, vocab.Count).ToDictionary(i => itob[i], i => i);
 var gpt = new GPT(vocabularySize: vocab.Count,
                      embeddingSize: 128,
@@ -111,6 +106,7 @@ double TrainOnFile(string filePath, ProgressBar parentProgressBar, out int batch
         ShowEstimatedDuration = true,
     };
     using var progressBar = parentProgressBar.Spawn(batches, "", displayOptions);
+    var stopwatch = Stopwatch.StartNew();
     for (int batchIndex = 0; batchIndex < batches; batchIndex++) {
         using var _ = torch.NewDisposeScope();
         var (@in, @out) = GetBatch(batchIndex);
@@ -131,7 +127,10 @@ double TrainOnFile(string filePath, ProgressBar parentProgressBar, out int batch
         using var noGrad = no_grad();
         totalLoss += loss.detach().cpu().mean().ToDouble();
 
-        progressBar.Tick($"loss: {totalLoss / (batchIndex + 1):0.00}");
+        int tokensPerSecond = (int)(step * batchSize / stopwatch.Elapsed.TotalSeconds);
+
+        progressBar.Tick($"loss: {totalLoss / (batchIndex + 1):0.00}" +
+            $" {tokensPerSecond} tokens/s");
     }
 
     return totalLoss;

app/app.csproj

@@ -13,9 +13,9 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="libtorch-cuda-11.7-win-x64" Version="1.13.0.1" />
     <PackageReference Include="ShellProgressBar" Version="5.2.0" />
-    <PackageReference Include="System.Drawing.Common" Version="6.0.0" />
+    <PackageReference Include="System.Drawing.Common" Version="8.0.8" />
+    <PackageReference Include="TorchSharp-cuda-windows" Version="0.103.0" />
   </ItemGroup>
 
   <ItemGroup>

src/GPT.cs

@@ -56,6 +56,8 @@ public GPT(int vocabularySize, int blockSize, int blockCount, int embeddingSize,
         this.Register(out this.finalNorm, LayerNorm(new long[] { embeddingSize }));
         this.Register(out this.decoderHead, Linear(embeddingSize, vocabularySize, hasBias: false));
 
+        this.RegisterComponents();
+
         this.apply(InitWeights);
     }
 

src/MinGPT.csproj

@@ -39,8 +39,8 @@
 
   <ItemGroup>
     <!-- The following is recommended for public projects -->
-    <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1" PrivateAssets="All" />
-    <PackageReference Include="TorchSharp" Version="0.99.6" />
+    <PackageReference Include="Microsoft.SourceLink.GitHub" Version="8.0.0" PrivateAssets="All" />
+    <PackageReference Include="TorchSharp" Version="0.103.0" />
   </ItemGroup>
 
 </Project>

test/MinGPT.Tests.csproj

@@ -12,9 +12,9 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="libtorch-cpu" Version="1.13.0.1" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
-    <PackageReference Include="System.Drawing.Common" Version="6.0.0" />
+    <PackageReference Include="System.Drawing.Common" Version="8.0.8" />
+    <PackageReference Include="TorchSharp-cuda-windows" Version="0.103.0" />
     <PackageReference Include="xunit" Version="2.6.3" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.5.5">
       <PrivateAssets>all</PrivateAssets>