iterate

- Trying to fetch directly from the API is too dangerous at this point, since the API is blocked
- I've chosen to iterate on a GraphQL subtitles upload API
- Trying the GraphQL mutation with Absinthe, I got a "No query document supplied" error
	-
	  ```gql
	  mutation SetVideoCaptions($captions: Upload!) {
	    setVideoCaptions(videoId: 1, captions: $captions) {
	      captions {
	       text
	      }
	    }
	  }
	  ```
- Decided to upgrade Absinthe to see how that goes
- [absinthe_ecto](https://github.com/absinthe-graphql/absinthe_ecto) is deprecated, so I'm looking to move to [dataloader](https://github.com/absinthe-graphql/dataloader)
- Updated the dependencies, now looking to migrate the actual `assoc` calls

Author: Betree

apps/cf/lib/videos/captions_fetcher_youtube.ex

@@ -69,30 +69,7 @@ defmodule CF.Videos.CaptionsFetcherYoutube do
   end
 
   defp process_transcript(transcript) do
-    transcript
-    |> SweetXml.xpath(
-      ~x"//transcript/text"l,
-      text: ~x"./text()"s |> transform_by(&clean_text/1),
-      start: ~x"./@start"s |> transform_by(&parse_float/1),
-      duration: ~x"./@dur"os |> transform_by(&parse_float/1)
-    )
-    |> Enum.filter(fn %{text: text, start: start} ->
-      start != nil and text != nil and text != ""
-    end)
-  end
-
-  defp clean_text(text) do
-    text
-    |> String.replace("&", "&")
-    |> HtmlEntities.decode()
-    |> String.trim()
-  end
-
-  defp parse_float(val) do
-    case Float.parse(val) do
-      {num, _} -> num
-      _ -> nil
-    end
+    CF.Videos.CaptionsSrv1Parser.parse_file(transcript)
   end
 
   # Below is an implementation using the official YouTube API, but it requires OAuth2 authentication.

apps/cf/lib/videos/captions_srv1_parser.ex

@@ -0,0 +1,37 @@
+defmodule CF.Videos.CaptionsSrv1Parser do
+  @moduledoc """
+  A captions parser for the srv1 format.
+  """
+
+  require Logger
+  import SweetXml
+
+  def parse_file(content) do
+    content
+    |> SweetXml.xpath(
+      ~x"//transcript/text"l,
+      text: ~x"./text()"s |> transform_by(&clean_text/1),
+      start: ~x"./@start"s |> transform_by(&parse_float/1),
+      duration: ~x"./@dur"os |> transform_by(&parse_float/1)
+    )
+    |> Enum.filter(fn %{text: text, start: start} ->
+      # Filter out text in brackets, like "[Music]"
+      start != nil and text != nil and text != "" and
+        String.match?(text, ~r/^\[.*\]$/) == false
+    end)
+  end
+
+  defp clean_text(text) do
+    text
+    |> String.replace("&", "&")
+    |> HtmlEntities.decode()
+    |> String.trim()
+  end
+
+  defp parse_float(val) do
+    case Float.parse(val) do
+      {num, _} -> num
+      _ -> nil
+    end
+  end
+end

apps/cf_graphql/lib/endpoint.ex

@@ -13,9 +13,9 @@ defmodule CF.GraphQLWeb.Endpoint do
 
   plug(
     Plug.Parsers,
-    parsers: [:urlencoded, :multipart, :json],
+    parsers: [:urlencoded, :multipart, :json, Absinthe.Plug.Parser],
     pass: ["*/*"],
-    json_decoder: Poison
+    json_decoder: Jason
   )
 
   plug(Plug.MethodOverride)

apps/cf_graphql/lib/resolvers/videos.ex

@@ -8,6 +8,7 @@ defmodule CF.Graphql.Resolvers.Videos do
   import Ecto.Query
   import Absinthe.Resolution.Helpers, only: [batch: 3]
 
+  alias Ecto.Multi
   alias DB.Repo
   alias DB.Schema.Video
   alias DB.Schema.VideoCaption
@@ -115,7 +116,6 @@ defmodule CF.Graphql.Resolvers.Videos do
     {:ok, video}
   end
 
-
   def edit(_root, %{id: id, unlisted: unlisted}, %{
         context: %{user: user}
       }) do
@@ -137,32 +137,55 @@ defmodule CF.Graphql.Resolvers.Videos do
       {:error, _} ->
         {:error, "Failed to update video"}
     end
+  end
 
   def set_captions(_root, %{video_id: video_id, captions: captions_input}, %{
         context: %{user: user}
       }) do
     video = DB.Repo.get!(DB.Schema.Video, video_id)
 
-    Multi.new()
-    |> Multi.insert(
-      :caption,
-      VideoCaption.changeset(%VideoCaption{
-        video_id: video.id,
-        raw: captions_input,
-        parsed: captions_input,
-        format: "user-provided"
-      })
-    )
-    |> Multi.run(
-      :action,
-      fn _repo, %{caption: caption} ->
-        CF.Actions.ActionCreator.action_upload_video_captions(user.id, video.id, caption)
-        |> DB.Repo.insert!()
-
-        {:ok, caption}
+    def set_captions(
+          _root,
+          %{video_id: video_id, captions: %{path: path, content_type: content_type}},
+          %{
+            context: %{user: user}
+          }
+        ) do
+      cond do
+        content_type != "text/xml" ->
+          {:error, "Unsupported content type: #{content_type}"}
+
+        File.stat!(path).size > 10_000_000 ->
+          {:error, "File must be < 10MB"}
+
+        true ->
+          video = DB.Repo.get!(DB.Schema.Video, video_id)
+          captions_file_content = File.read!(path)
+          parsed = CF.Videos.CaptionsSrv1Parser.parse_file(captions_file_content)
+
+          Multi.new()
+          |> Multi.insert(
+            :caption,
+            VideoCaption.changeset(%VideoCaption{
+              video_id: video.id,
+              raw: captions_file_content,
+              parsed: parsed,
+              format: "xml"
+            })
+          )
+          |> Multi.run(
+            :action,
+            fn _repo, %{caption: caption} ->
+              CF.Actions.ActionCreator.action_upload_video_captions(user.id, video.id, caption)
+              |> DB.Repo.insert!()
+
+              {:ok, caption}
+            end
+          )
+          |> DB.Repo.transaction()
+
+          {:ok, video}
       end
-    )
-
-    {:ok, video}
+    end
   end
 end

apps/cf_graphql/lib/schema/input_objects.ex

@@ -4,7 +4,6 @@ defmodule CF.Graphql.Schema.InputObjects.StatementFilter do
   """
 
   use Absinthe.Schema.Notation
-  use Absinthe.Ecto, repo: DB.Repo
 
   @desc "Props to filter statements on"
   input_object :statement_filter do

apps/cf_graphql/lib/schema/input_objects/statement_filter.ex

@@ -4,7 +4,6 @@ defmodule CF.Graphql.Schema.InputObjects.VideoFilter do
   """
 
   use Absinthe.Schema.Notation
-  use Absinthe.Ecto, repo: DB.Repo
 
   @desc "Props to filter videos on"
   input_object :video_filter do

apps/cf_graphql/lib/schema/input_objects/video_caption.ex

@@ -2,9 +2,42 @@ defmodule CF.Graphql.Schema do
   use Absinthe.Schema
   alias CF.Graphql.Resolvers
   alias CF.Graphql.Schema.Middleware
+  import Absinthe.Resolution.Helpers, only: [dataloader: 1]
+
+  import_types(Absinthe.Plug.Types)
+
+  import_types(CF.Graphql.Schema.Types.{
+    AppInfo,
+    Comment,
+    Notification,
+    Paginated,
+    Source,
+    Speaker,
+    Statement,
+    Statistics,
+    Subscription,
+    UserAction,
+    User,
+    Video,
+    VideoCaption
+  })
+
+  import_types(CF.Graphql.Schema.InputObjects.{
+    VideoFilter,
+    StatementFilter
+  })
+
+  def context(ctx) do
+    loader =
+      Dataloader.new()
+      |> Dataloader.add_source(DB.Repo, Dataloader.Ecto.new(DB.Repo))
+
+    Map.put(ctx, :loader, loader)
+  end
 
-  import_types(CF.Graphql.Schema.Types)
-  import_types(CF.Graphql.Schema.InputObjects)
+  def plugins do
+    [Absinthe.Middleware.Dataloader | Absinthe.Plugin.defaults()]
+  end
 
   # Query API
 
@@ -109,16 +142,16 @@ defmodule CF.Graphql.Schema do
       arg(:unlisted, non_null(:boolean))
 
       resolve(&Resolvers.Videos.edit/3)
-         end
+    end
 
     field :set_video_captions, :video do
       middleware(Middleware.RequireAuthentication)
       middleware(Middleware.RequireReputation, 450)
 
       arg(:video_id, non_null(:id))
-      arg(:captions, non_null(list_of(:video_caption_input)))
+      arg(:captions, non_null(:upload))
 
       resolve(&Resolvers.Videos.set_captions/3)
-
+    end
   end
 end