♻️ 💥 change raw_http attribute in responses to be actual json strings

Author: sebastianMindee

bin/v1/parser.rb

@@ -42,9 +42,11 @@ def init_product_parser
       V1_PRODUCTS.each do |doc_key, doc_value|
         v1_product_parser[doc_key] = OptionParser.new do |options_parser|
           options_parser.on('-w', '--all-words', 'Include words in response') { |v| @options[:all_words] = v }
-          options_parser.on('-c', '--cut-pages', 'Cut document pages') { |v| @options[:cut_pages] = v }
           options_parser.on('-k [KEY]', '--key [KEY]', 'API key for the endpoint') { |v| @options[:api_key] = v }
-          options_parser.on('-f', '--full', 'Print the full data') { @options[:print_full] = true }
+          options_parser.on('-o FORMAT', '--output-format FORMAT', ['raw', 'full', 'summary'],
+                            'Format of the output (raw, full, summary). Default: summary') do |format|
+            @options[:output_format] = format
+          end
           options_parser.on('-F', '--fix-pdf', 'Repair PDF') { @options[:repair_pdf] = true }
 
           if doc_key != 'universal'
@@ -69,23 +71,27 @@ def send(product_command, endpoint_name, options)
       doc_class = V1_PRODUCTS[product_command][:doc_class]
       input_source = setup_input_source(mindee_client, options)
       custom_endpoint = setup_endpoint(mindee_client, product_command, endpoint_name, options)
-      page_options = setup_page_options(options)
       options[:parse_async] = !V1_PRODUCTS[product_command][:sync] if options[:parse_async].nil?
 
       mindee_client.parse(
         input_source,
         doc_class,
         options: { endpoint: custom_endpoint,
-                   options: Mindee::ParseOptions.new(
-                     params: { page_options: page_options }
-                   ),
                    enqueue: options[:parse_async] }
       )
     end
 
+    def print_result(result, output_format)
+      if output_format == :raw
+        puts JSON.pretty_generate(JSON.parse(result.raw_http))
+      else
+        puts output_format == :full ? result.document : result.document.inference.prediction
+      end
+    end
+
     # @param product_command [String]
     def execute
-      options = {}
+      @options = { output_format: :summary }
       product_command = @arguments.shift
 
       abort(@options_parser.help) unless V1_PRODUCTS.include?(product_command)
@@ -97,23 +103,27 @@ def execute
           abort(@product_parser[product_command].help)
         end
         endpoint_name = @arguments[0]
-        options[:file_path] = @arguments[1]
+        @options[:file_path] = @arguments[1]
       else
         if @arguments.empty?
           warn 'file missing'
           abort(@product_parser[product_command].help)
         end
         endpoint_name = nil
-        options[:file_path] = @arguments[0]
+        @options[:file_path] = @arguments[0]
       end
 
-      result = send(product_command, endpoint_name, options)
-
-      puts options[:print_full] ? result.document : result.document.inference.prediction
+      result = send(product_command, endpoint_name, @options)
+      print_result(result, output_format)
     end
 
     private
 
+    # @return [Symbol]
+    def output_format
+      @options[:output_format]&.to_sym || :summary
+    end
+
     # @param mindee_client [Mindee::V1::Client]
     # @param options [Hash]
     # @return [Hash]
@@ -139,22 +149,5 @@ def setup_endpoint(mindee_client, product_command, endpoint_name, options)
         version: options[:endpoint_version] || '1'
       )
     end
-
-    # @param options [Hash]
-    # @return [Hash]
-    def setup_page_options(options)
-      if options[:cut_pages].nil? || !options[:cut_pages].is_a?(Integer) ||
-         options[:cut_pages].negative?
-        nil
-      else
-
-        { params: {
-          page_indexes: (0..options[:cut_pages].to_i).to_a,
-          operation: :KEEP_ONLY,
-          on_min_pages: 0,
-        } }
-
-      end
-    end
   end
 end

bin/v2/parser.rb

@@ -28,41 +28,39 @@ def initialize(arguments)
       @search_parser = init_search_parser
     end
 
-    # Summarize the result of the command.
+    # Summarize and print the result of the command.
     # @param command [String]
-    # @return [String]
-    def summarize_result(command)
+    def print_result(command)
       if command == 'search-models'
         @search_parser.parse!(@arguments)
-        @result = search(@options)
-        summarized_result = @options[:print_full] ? @result.to_s : @result.models.to_s
+        result = search(@options)
+        summarized_result = output_format == :full ? result.to_s : result.models.to_s
       else
         @product_parser[command].parse!(@arguments)
         @options[:file_path] = @arguments.shift
         if @options[:file_path].nil?
           warn 'file missing'
           abort(@product_parser[command].help)
         end
-        @result = send(command, @options)
-        summarized_result = @options[:print_full] ? @result.inference.to_s : @result.inference.result.to_s
+        result = send(command, @options)
+        summarized_result = output_format == :full ? result.inference.to_s : result.inference.result.to_s
+      end
+
+      if output_format == :raw
+        puts JSON.pretty_generate(raw_payload(result.raw_http))
+      else
+        puts summarized_result
       end
-      summarized_result
     end
 
     # Executes the command.
     # @return [void]
     def execute
-      @options = {}
+      @options = { output_format: :summary }
       command = @arguments.shift
 
       validate_command!(command)
-      summarized_result = summarize_result(command)
-
-      if @options[:raw_json]
-        puts JSON.pretty_generate(@result.raw_http)
-      else
-        puts summarized_result
-      end
+      print_result(command)
     rescue OptionParser::InvalidOption, OptionParser::MissingArgument => e
       if command == 'search-models'
         abort("#{e.message}\n\n#{@search_parser.help}")
@@ -101,24 +99,24 @@ def init_search_parser
     end
 
     def setup_specific_options(options_parser, doc_value)
-      options_parser.on('-r', '--rag', 'Enable RAG') { @options[:rag] = true } if doc_value[:rag]
-      if doc_value[:raw_text]
+      options_parser.on('-r', '--rag', 'Enable RAG') { @options[:rag] = true } if doc_value.key?(:rag)
+      if doc_value.key?(:raw_text)
         options_parser.on('-R', '--raw-text', 'Enable Raw Text retrieval') do
           @options[:raw_text] = true
         end
       end
-      if doc_value[:confidence]
+      if doc_value.key?(:confidence)
         options_parser.on('-c', '--confidence', 'Enable confidence scores') do
           @options[:confidence] = true
         end
       end
-      options_parser.on('-p', '--polygon', 'Enable polygons') { @options[:polygon] = true } if doc_value[:polygon]
-      if doc_value[:text_context]
+      options_parser.on('-p', '--polygon', 'Enable polygons') { @options[:polygon] = true } if doc_value.key?(:polygon)
+      if doc_value.key?(:text_context)
         options_parser.on('-t [TEXT CONTEXT]', '--text-context [TEXT CONTEXT]', 'Add Text Context') do |v|
           @options[:text_context] = v
         end
       end
-      return unless doc_value[:data_schema]
+      return unless doc_value.key?(:data_schema)
 
       options_parser.on('-d [DATA SCHEMA]', '--data-schema [DATA SCHEMA]', 'Add Data Schema') do |v|
         @options[:data_schema] = v
@@ -129,8 +127,31 @@ def setup_specific_options(options_parser, doc_value)
     # @param options_parser [OptionParser]
     def init_common_options(options_parser)
       options_parser.on('-k [KEY]', '--key [KEY]', 'API key for the endpoint') { |v| @options[:api_key] = v }
-      options_parser.on('-f', '--full', 'Print the full data') { @options[:print_full] = true }
-      options_parser.on('-j', '--raw-json', 'Print the full raw jason data') { @options[:raw_json] = true }
+      options_parser.on('-o FORMAT', '--output-format FORMAT', ['raw', 'full', 'summary'],
+                        'Format of the output (raw, full, summary). Default: summary') do |format|
+        @options[:output_format] = format
+      end
+    end
+
+    # @return [Symbol]
+    def output_format
+      @options[:output_format]&.to_sym || :summary
+    end
+
+    # Handles JSON payloads represented either as a string or an already-parsed hash.
+    # Also tolerates one extra JSON encoding layer.
+    # @param payload [String, Hash]
+    # @return [Hash, Array, String]
+    def raw_payload(payload)
+      parsed_payload = payload
+      2.times do
+        break unless parsed_payload.is_a?(String)
+
+        parsed_payload = JSON.parse(parsed_payload)
+      rescue JSON::ParserError
+        break
+      end
+      parsed_payload
     end
 
     # @return [Hash]
@@ -144,19 +165,21 @@ def init_product_parser
             @options[:alias] = v
           end
           init_common_options(options_parser)
-          options_parser.on('-C PAGES', '--cut-pages PAGES', 'Cut document pages') { |v| @options[:cut_pages] = v }
-          options_parser.on('-F', '--fix-pdf', 'Repair PDF') { @options[:repair_pdf] = true }
+          options_parser.on('-F', '--fix-pdf', 'Attempt to repair PDF before enqueueing') do
+            @options[:repair_pdf] = true
+          end
           setup_specific_options(options_parser, product_values)
         end
       end
       v2_product_parser
     end
 
-    # @param options [Hash] General options.
-    # @return page_options [Hash] Page options.
-    def setup_product_params(options, page_options)
-      params = { model_id: options[:model_id] }
-      params[:options] = Mindee::ParseOptions.new(params: page_options) unless page_options.nil?
+    # @return [Hash]
+    def setup_product_params
+      params = { model_id: @options[:model_id] }
+      @options.each_pair do |key, value|
+        params[key] = value if V2_PRODUCTS['extraction'].include?(key)
+      end
       params
     end
 
@@ -167,8 +190,7 @@ def send(product_command, options)
       mindee_client = Mindee::ClientV2.new(api_key: options[:api_key])
       response_class = V2_PRODUCTS[product_command][:response_class]
       input_source = setup_input_source(options)
-      page_options = setup_page_options(options)
-      params = setup_product_params(options, page_options)
+      params = setup_product_params
 
       mindee_client.enqueue_and_get_result(
         response_class,

bin/v2/products.rb

@@ -8,51 +8,27 @@
   'classification' => {
     description: 'Classification Utility',
     response_class: Mindee::V2::Product::Classification::Classification,
-    rag: false,
-    polygon: false,
-    confidence: false,
-    raw_text: false,
-    text_context: false,
-    data_schema: false,
   },
   'extraction' => {
     description: 'Extraction Inference',
     response_class: Mindee::V2::Product::Extraction::Extraction,
-    rag: false,
-    polygon: false,
-    confidence: false,
-    raw_text: false,
-    text_context: false,
-    data_schema: false,
+    rag: true,
+    polygon: true,
+    confidence: true,
+    raw_text: true,
+    text_context: true,
+    data_schema: true,
   },
   'crop' => {
     description: 'Crop Utility',
     response_class: Mindee::V2::Product::Crop::Crop,
-    rag: false,
-    polygon: false,
-    confidence: false,
-    raw_text: false,
-    text_context: false,
-    data_schema: false,
   },
   'ocr' => {
     description: 'OCR Utility',
     response_class: Mindee::V2::Product::Ocr::Ocr,
-    rag: false,
-    polygon: false,
-    confidence: false,
-    raw_text: true,
-    text_context: false,
-    data_schema: false,
   },
   'split' => {
     description: 'Split Utility',
     response_class: Mindee::V2::Product::Split::Split,
-    rag: false,
-    polygon: false,
-    confidence: false,
-    raw_text: false,
-    text_context: false,
-    data_schema: false,
   },
 }.freeze

lib/mindee/client.rb

@@ -162,7 +162,7 @@ def parse_sync(input_source, product_class, endpoint, options)
         options
       )
 
-      Mindee::Parsing::Common::ApiResponse.new(product_class, prediction, raw_http.to_s)
+      Mindee::Parsing::Common::ApiResponse.new(product_class, prediction, raw_http)
     end
 
     # Enqueue a document for async parsing
@@ -200,7 +200,7 @@ def enqueue(input_source, product_class, endpoint: nil, options: {})
         input_source,
         opts
       )
-      Mindee::Parsing::Common::ApiResponse.new(product_class, prediction, raw_http.to_json)
+      Mindee::Parsing::Common::ApiResponse.new(product_class, prediction, raw_http)
     end
 
     # Parses a queued document
@@ -215,7 +215,7 @@ def parse_queued(job_id, product_class, endpoint: nil)
       endpoint = initialize_endpoint(product_class) if endpoint.nil?
       logger.debug("Fetching queued document as '#{endpoint.url_root}'")
       prediction, raw_http = endpoint.parse_async(job_id)
-      Mindee::Parsing::Common::ApiResponse.new(product_class, prediction, raw_http.to_json)
+      Mindee::Parsing::Common::ApiResponse.new(product_class, prediction, raw_http)
     end
 
     # Enqueue a document for async parsing and automatically try to retrieve it

lib/mindee/parsing/common/api_response.rb

@@ -39,10 +39,10 @@ class ApiResponse
 
         # @param product_class [Mindee::Inference]
         # @param http_response [Hash]
-        # @param raw_http [Hash]
+        # @param raw_http [String]
         def initialize(product_class, http_response, raw_http)
           logger.debug('Handling API response')
-          @raw_http = raw_http.to_s
+          @raw_http = raw_http
           raise Errors::MindeeAPIError, 'Invalid response format.' unless http_response.key?('api_request')
 
           @api_request = Mindee::Parsing::Common::ApiRequest.new(http_response['api_request'])

lib/mindee/v2/parsing/common_response.rb

@@ -1,16 +1,18 @@
 # frozen_string_literal: true
 
+require 'json'
+
 module Mindee
   module V2
     module Parsing
       # Base class for inference and job responses on the V2 API.
       class CommonResponse
-        # @return [Hash]
+        # @return [String]
         attr_reader :raw_http
 
         # @param http_response [Hash]
         def initialize(http_response)
-          @raw_http = http_response
+          @raw_http = JSON.generate(http_response)
         end
       end
     end

sig/mindee/parsing/common/api_response.rbs

@@ -15,11 +15,12 @@ module Mindee
 
 
       class ApiResponse
+        attr_reader document: Parsing::Common::Document?
+        attr_reader job: Parsing::Common::Job?
+        attr_reader api_request: Parsing::Common::ApiRequest
+        attr_reader raw_http: String
+
         def logger: () -> Logger
-        def document: -> Parsing::Common::Document?
-        def job: -> Parsing::Common::Job?
-        def api_request: -> Parsing::Common::ApiRequest?
-        def raw_http: -> String
         def initialize: (singleton(Parsing::Common::Inference), Hash[String | Symbol, untyped] | Net::HTTPResponse, String) -> void
       end
     end