Class: Metanorma::Compile

Inherits:
Object
  • Object
show all
Includes:
CompileOptions, Flavor, Validator, Writeable
Defined in:
lib/metanorma/compile/flavor.rb,
lib/metanorma/compile/render.rb,
lib/metanorma/compile/compile.rb,
lib/metanorma/compile/extract.rb,
lib/metanorma/compile/validator.rb,
lib/metanorma/compile/writeable.rb,
lib/metanorma/compile/relaton_drop.rb,
lib/metanorma/compile/compile_options.rb,
lib/metanorma/compile/output_filename.rb,
lib/metanorma/compile/output_filename_config.rb

Defined Under Namespace

Modules: CompileOptions, Extract, Flavor, Validator, Writeable Classes: OutputFilename, OutputFilenameConfig, RelatonDrop

Constant Summary collapse

DEFAULT_NUM_WORKERS = 
3

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from Writeable

#export_output

Methods included from Flavor

#load_flavor, #stdtype2flavor_gem

Methods included from CompileOptions

#extract_dir_options, #extract_extension_options, #extract_extensions, #extract_flavor_options, #extract_options, #extract_xml_options, #font_install, #get_extensions, #options_in_file, #require_libraries

Methods included from Validator

#validate_format!, #validate_options!, #validate_type!

Constructor Details

#initialize ⇒ Compile

Returns a new instance of Compile.



37
38
39
40
41
42
43
 
# File 'lib/metanorma/compile/compile.rb', line 37

def initialize
  @registry = Metanorma::Registry.instance
  @errors = []
  @isodoc = IsoDoc::Convert.new({})
  @fontist_installed = false
  @log = Metanorma::Utils::Log.new
end

Instance Attribute Details

#errors ⇒ Array<String> (readonly)

Returns:



34
35
36
 
# File 'lib/metanorma/compile/compile.rb', line 34

def errors
  @errors
end

#processor ⇒ Object (readonly)

Returns the value of attribute processor.



35
36
37
 
# File 'lib/metanorma/compile/compile.rb', line 35

def processor
  @processor
end

Instance Method Details

#clean_exit(options) ⇒ Object 



86
87
88
89
 
# File 'lib/metanorma/compile/compile.rb', line 86

def clean_exit(options)
  options[:novalid] and return
  @log.write
end

#compile(filename, options = {}) ⇒ Object

Main compile method that orchestrates the document conversion process

Parameters:

  • filename (String) —

    path to the input file

  • options (Hash) (defaults to: {}) —

    compilation options



48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
 
# File 'lib/metanorma/compile/compile.rb', line 48

def compile(filename, options = {})
  process_options!(filename, options)
  @processor = @registry.find_processor(options[:type].to_sym)

  # Step 1: Generate Semantic XML
  semantic_result = generate_semantic_xml(filename, options) or return nil
  source_file, semantic_xml = semantic_result

  # Step 2: Prepare output paths
  xml = Nokogiri::XML(semantic_xml, &:huge)
  bibdata = extract_relaton_metadata(xml)
  output_paths = prepare_output_paths(filename, bibdata, options)

  # Step 3: Determine which output formats to generate
  extensions = get_extensions(options)
  @extensions = extensions # Capture extensions for later access
  extensions or return nil

  # Step 4: Extract information from Semantic XML if requested
  extract_information(semantic_xml, bibdata, options)

  # Step 5: Generate output formats from Semantic XML
  generate_outputs(
    source_file, semantic_xml, bibdata, extensions, output_paths,
    options
  )
ensure
  clean_exit(options)
end

#export_relaton_from_bibdata(bibdata, options) ⇒ Object

Export given bibliographic data to Relaton XML on disk

Parameters:

  • bibdata (Nokogiri::XML::Element) —

    the bibliographic data element

  • options (Hash) —

    compilation options



37
38
39
40
 
# File 'lib/metanorma/compile/render.rb', line 37

def export_relaton_from_bibdata(bibdata, options)
  options[:relaton] or return
  export_output(options[:relaton], bibdata.to_xml)
end

#extract_information(semantic_xml, bibdata, options) ⇒ Object

Step 4: Extract information from Semantic XML

Parameters:

  • semantic_xml (String) —

    semantic XML content

  • options (Hash) —

    compilation options



134
135
136
137
138
139
140
141
142
143
 
# File 'lib/metanorma/compile/compile.rb', line 134

def extract_information(semantic_xml, bibdata, options)
  # Extract Relaton bibliographic data
  export_relaton_from_bibdata(bibdata, options) if options[:relaton]
  # Extract other components (sourcecode, images, requirements)
  if options[:extract]
    Extract.extract(
      semantic_xml, options[:extract], options[:extract_type]
    )
  end
end

#extract_log_messages(flavor) ⇒ Object 



254
255
256
257
 
# File 'lib/metanorma/compile/compile.rb', line 254

def extract_log_messages(flavor)
  process_compile_dummy(flavor)
  @log.display_messages
end

#extract_relaton_metadata(xml) ⇒ Nokogiri::XML::Element

Returns the bibliographic data element.

Parameters:

  • xml (Nokogiri::XML::Document) —

    the XML document

Returns:

  • (Nokogiri::XML::Element) —

    the bibliographic data element



44
45
46
 
# File 'lib/metanorma/compile/render.rb', line 44

def extract_relaton_metadata(xml)
  xml.at("//bibdata") || xml.at("//xmlns:bibdata")
end

#gather_and_install_fonts(source_file, options, extensions) ⇒ Object

isodoc is Raw Metanorma XML



58
59
60
61
62
63
 
# File 'lib/metanorma/compile/render.rb', line 58

def gather_and_install_fonts(source_file, options, extensions)
  Util.sort_extensions_execution(extensions).each do |ext|
    isodoc_options = get_isodoc_options(source_file, options, ext)
    font_install(isodoc_options.merge(options))
  end
end

#generate_outputs(source_file, semantic_xml, bibdata, extensions, output_paths, options) ⇒ Object

Step 5: Generate output formats from Semantic XML

Parameters:

  • source_file (String) —

    source file content

  • semantic_xml (String) —

    semantic XML content

  • bibdata (Nokogiri::XML::Element) —

    the bibliographic data element

  • extensions (Array<Symbol>) —

    output formats to generate

  • output_paths (Hash) —

    paths for output files

  • options (Hash) —

    compilation options



152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
 
# File 'lib/metanorma/compile/compile.rb', line 152

def generate_outputs(
  source_file, semantic_xml, bibdata, extensions, output_paths, options
)
  if extensions == i(presentation)
    # Just generate presentation XML
    generate_presentation_xml(
      source_file, semantic_xml, bibdata, output_paths, options
    )
  else
    # Generate multiple output formats with parallel processing
    generate_outputs_parallel(
      source_file, semantic_xml, bibdata, extensions, output_paths, options
    )
  end
end

#generate_outputs_parallel(source_file, semantic_xml, bibdata, extensions, output_paths, options) ⇒ Object

Generate multiple output formats with parallel processing



9
10
11
12
13
14
15
16
17
18
19
20
21
22
 
# File 'lib/metanorma/compile/render.rb', line 9

def generate_outputs_parallel(
  source_file, semantic_xml, bibdata, extensions, output_paths, options
)
  @queue = ::Metanorma::Util::WorkersPool.new(
    ENV["METANORMA_PARALLEL"]&.to_i || DEFAULT_NUM_WORKERS,
  )
  # Install required fonts for all extensions
  gather_and_install_fonts(source_file, options.dup, extensions)
  # Process each extension in order
  process_extensions_in_order(
    source_file, semantic_xml, bibdata, extensions, output_paths, options
  )
  @queue.shutdown
end

#generate_presentation_xml(source_file, xml, bibdata, output_paths, opt) ⇒ Object

Generate presentation XML from semantic XML



4
5
6
 
# File 'lib/metanorma/compile/render.rb', line 4

def generate_presentation_xml(source_file, xml, bibdata, output_paths, opt)
  process_ext(:presentation, source_file, xml, bibdata, output_paths, opt)
end

#generate_semantic_xml(filename, options) ⇒ Array?

Step 1: Generate Semantic XML from input file

Parameters:

  • filename (String) —

    input file path

  • options (Hash) —

    compilation options

Returns:

  • (Array, nil) —

    tuple of [source_file, semantic_xml] or nil on failure



95
96
97
98
99
100
101
102
103
104
 
# File 'lib/metanorma/compile/compile.rb', line 95

def generate_semantic_xml(filename, options)
  case extname = File.extname(filename)
  when ".adoc" then process_input_adoc(filename, options)
  when ".xml" then process_input_xml(filename, options)
  else
    Util.log("[metanorma] Error: file extension #{extname} " \
             "is not supported.", :error)
    nil
  end
end

#output_format_mapping ⇒ Hash

Returns a mapping of format types to file suffixes based on the extensions that were determined during compilation. This method is only available after compile() has been called and reached Step 3 where extensions are finalized.

Returns:

  • (Hash) —

    mapping of format symbols to file extensions e.g., { html: “html”, doc: “doc”, pdf: “pdf” }

  • (Hash) —

    empty hash if extensions haven’t been captured yet



233
234
235
236
237
238
239
240
 
# File 'lib/metanorma/compile/compile.rb', line 233

def output_format_mapping
  @extensions && @processor or return {}
  @extensions.each_with_object({}) do |ext, mapping|
    if @processor.output_formats[ext]
      mapping[ext] = @processor.output_formats[ext]
    end
  end
end

#prepare_output_paths(filename, bibdata, options) ⇒ Hash

Step 2: Prepare output paths for generated files Use default filename template if empty string is provided.

Parameters:

  • filename (String) —

    input file path

  • bibdata (Nokogiri::XML::Element) —

    the bibliographic data element

  • options (Hash) —

    compilation options

Returns:

  • (Hash) —

    paths for different output formats



113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
 
# File 'lib/metanorma/compile/compile.rb', line 113

def prepare_output_paths(filename, bibdata, options)
  basename = if options[:filename_template].nil?
               filename.sub(/\.[^.]+$/, "")
             else
               drop = RelatonDrop.new(bibdata)
               config = OutputFilenameConfig.new(options[:filename_template])
               config.generate_filename(drop)
             end
  @output_filename = OutputFilename.new(
    basename, options[:output_dir], @processor
  )
  {
    xml: @output_filename.semantic_xml,
    orig_filename: filename,
    presentationxml: @output_filename.presentation_xml,
  }
end

#process_compile_dummy(flavor) ⇒ Object 



242
243
244
245
246
247
248
249
250
251
252
 
# File 'lib/metanorma/compile/compile.rb', line 242

def process_compile_dummy(flavor)
  file = Tempfile.new(%w(tmp .adoc))
  file.write "    Document Title\n    :flavor: \#{flavor}\n    :mn-output-extensions: xml, presentation\n\n  OUTPUT\n  file.close\n  compile(file.path, agree_to_terms: true)\nend\n"

#process_ext(ext, source_file, semantic_xml, bibdata, output_paths, options) ⇒ Object

Process a single extension (output format)



66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
 
# File 'lib/metanorma/compile/render.rb', line 66

def process_ext(ext, source_file, semantic_xml, bibdata, output_paths,
                options)
  output_paths[:ext] = @processor.output_formats[ext]
  output_paths[:out] = @output_filename.for_format(ext) ||
    output_paths[:xml].sub(/\.[^.]+$/, ".#{output_paths[:ext]}")
  isodoc_options = get_isodoc_options(source_file, options, ext)

  # Handle special cases first
  return true if process_ext_special(
    ext, semantic_xml, bibdata, output_paths, options, isodoc_options
  )

  # Otherwise, determine if it uses presentation XML
  if @processor.use_presentation_xml(ext)
    # Format requires presentation XML first, then convert to final format
    process_via_presentation_xml(ext, output_paths, options, isodoc_options)
  else
    # Format can be generated directly from semantic XML
    process_from_semantic_xml(
      ext, output_paths, semantic_xml, isodoc_options
    )
  end
end

#process_ext_special(ext, sem_xml, bibdata, output_paths, options, isodoc_options) ⇒ Object

Process special extensions with custom handling



91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
 
# File 'lib/metanorma/compile/render.rb', line 91

def process_ext_special(
  ext, sem_xml, bibdata, output_paths, options, isodoc_options
)
  if ext == :rxl
    # Special case: Relaton export
    export_relaton_from_bibdata(
      bibdata,
      options.merge(relaton: output_paths[:out]),
    )
    true
  elsif ext == :presentation && options[:passthrough_presentation_xml]
    # Special case: Pass through presentation XML
    f = if File.exist?(output_paths[:orig_filename])
          output_paths[:orig_filename]
        else
          output_paths[:xml]
        end
    FileUtils.cp f, output_paths[:presentationxml]
    true
  elsif ext == :html && options[:sectionsplit]
    # Special case: Split HTML into sections
    sectionsplit_convert(
      output_paths[:xml], sem_xml, output_paths[:out], isodoc_options
    )
    true
  else
    false
  end
end

#process_extensions_in_order(source_file, semantic_xml, bibdata, extensions, output_paths, options) ⇒ Object 



24
25
26
27
28
29
30
31
32
 
# File 'lib/metanorma/compile/render.rb', line 24

def process_extensions_in_order(
  source_file, semantic_xml, bibdata, extensions, output_paths, options
)
  Util.sort_extensions_execution(extensions).each do |ext|
    process_ext(
      ext, source_file, semantic_xml, bibdata, output_paths, options
    ) or break
  end
end

#process_from_semantic_xml(ext, output_paths, sem_xml, isodoc_options) ⇒ Object

Process format directly from semantic XML



141
142
143
144
145
146
147
148
149
 
# File 'lib/metanorma/compile/render.rb', line 141

def process_from_semantic_xml(ext, output_paths, sem_xml, isodoc_options)
  @processor.output(sem_xml, output_paths[:xml], output_paths[:out],
                    ext, isodoc_options)
  true # Return as Thread equivalent
rescue StandardError => e
  strict = ext == :presentation || isodoc_options[:strict] == "true"
  isodoc_error_process(e, strict, true)
  ext != :presentation
end

#process_input_adoc(filename, options) ⇒ Object 



191
192
193
194
195
196
197
198
 
# File 'lib/metanorma/compile/compile.rb', line 191

def process_input_adoc(filename, options)
  Util.log("[metanorma] Processing: AsciiDoc input.", :info)
  file = read_file(filename)
  file = process_input_adoc_hdr(file, options)
  file = process_input_adoc_includes(file, filename)
  update_options_after_preprocessing(file, options)
  [file, @processor.input_to_isodoc(file, filename, options)]
end

#process_input_adoc_hdr(file, options) ⇒ Object 



168
169
170
171
172
173
174
 
# File 'lib/metanorma/compile/compile.rb', line 168

def process_input_adoc_hdr(file, options)
  hdr, rest = Metanorma::Input::Asciidoc.new.header(file)
  attrs = hdr.split("\n")
  options[:asciimath] and attrs << ":mn-keep-asciimath:"
  process_input_adoc_overrides(attrs, options)
  "#{attrs.join("\n")}\n\n#{rest}"
end

#process_input_adoc_includes(file, filename) ⇒ Object 



183
184
185
186
187
188
189
 
# File 'lib/metanorma/compile/compile.rb', line 183

def process_input_adoc_includes(file, filename)
  dir = File.dirname(filename)
  dir != "." and
    file = file.gsub(/^include::/, "include::#{dir}/")
      .gsub(/^embed::/, "embed::#{dir}/")
  file
end

#process_input_adoc_overrides(attrs, options) ⇒ Object 



176
177
178
179
180
181
 
# File 'lib/metanorma/compile/compile.rb', line 176

def process_input_adoc_overrides(attrs, options)
  Metanorma::TasteRegister.instance.available_tastes
    .include?(options[:supplied_type]) or return
  c = Metanorma::TasteRegister.get(options[:supplied_type])
  c.process_input_adoc_overrides(attrs, options)
end

#process_input_xml(filename, _options) ⇒ Object 



214
215
216
217
218
219
 
# File 'lib/metanorma/compile/compile.rb', line 214

def process_input_xml(filename, _options)
  Util.log("[metanorma] Processing: Metanorma XML input.", :info)
  # TODO NN: this is a hack -- we should provide/bridge the
  # document attributes in Metanorma XML
  ["", read_file(filename)]
end

#process_options!(filename, options) ⇒ Object 



78
79
80
81
82
83
84
 
# File 'lib/metanorma/compile/compile.rb', line 78

def process_options!(filename, options)
  require_libraries(options)
  options = extract_options(filename, options)
  validate_options!(options)
  @log.save_to(filename, options[:output_dir])
  options[:log] = @log
end

#process_output_from_presentation_xml(ext, output_paths, options, isodoc_options) ⇒ Object

Generate output format from presentation XML



130
131
132
133
134
135
136
137
138
 
# File 'lib/metanorma/compile/render.rb', line 130

def process_output_from_presentation_xml(ext, output_paths, options,
                                         isodoc_options)
  @processor.output(nil, output_paths[:presentationxml],
                    output_paths[:out], ext, isodoc_options)
  wrap_html(options, output_paths[:ext], output_paths[:out])
rescue StandardError => e
  strict = ext == :presentation || isodoc_options[:strict] == true
  isodoc_error_process(e, strict, false)
end

#process_via_presentation_xml(ext, output_paths, options, isodoc_options) ⇒ Object

Process format that requires presentation XML



122
123
124
125
126
127
 
# File 'lib/metanorma/compile/render.rb', line 122

def process_via_presentation_xml(ext, output_paths, options, isodoc_options)
  @queue.schedule(ext, output_paths.dup, options.dup,
                  isodoc_options.dup) do |a, b, c, d|
                    process_output_from_presentation_xml(a, b, c, d)
                  end
end

#read_file(filename) ⇒ Object 



221
222
223
 
# File 'lib/metanorma/compile/compile.rb', line 221

def read_file(filename)
  File.read(filename, encoding: "utf-8").gsub("\r\n", "\n")
end

#sectionsplit_convert(input_filename, file, output_filename = nil, opts = {}) ⇒ Object

assume we pass in Presentation XML, but we want to recover Semantic XML



152
153
154
155
156
157
158
159
160
161
162
163
164
 
# File 'lib/metanorma/compile/render.rb', line 152

def sectionsplit_convert(input_filename, file, output_filename = nil,
                         opts = {})
  @isodoc ||= IsoDoc::PresentationXMLConvert.new({})
  input_filename += ".xml" unless input_filename.match?(/\.xml$/)
  File.exist?(input_filename) or export_output(input_filename, file)
  presxml = File.read(input_filename, encoding: "utf-8")
  _xml, filename, dir = @isodoc.convert_init(presxml, input_filename, false)
  ::Metanorma::Collection::Sectionsplit.new(
    input: input_filename, isodoc: @isodoc, xml: presxml,
    base: File.basename(output_filename || filename),
    output: output_filename || filename, dir: dir, compile_opts: opts
  ).build_collection
end

#update_options_after_preprocessing(file, options) ⇒ Object

need to update options, for any content added; e.g. new attributes from Metanorma taste



202
203
204
205
206
207
208
209
210
211
212
 
# File 'lib/metanorma/compile/compile.rb', line 202

def update_options_after_preprocessing(file, options)
  new_options = { extension_keys: options[:extension_keys] }
  Tempfile.open(["tmp", ".adoc"], encoding: "UTF-8") do |f|
    f.write(file)
    f.close
    new_options = extract_options(f, new_options)
  end
  new_options.compact!
  new_options.delete(:filename)
  options.merge!(new_options)
end

#wrap_html(options, file_extension, outfilename) ⇒ Object 



48
49
50
51
52
53
54
55
 
# File 'lib/metanorma/compile/render.rb', line 48

def wrap_html(options, file_extension, outfilename)
  if options[:wrapper] && /html$/.match(file_extension)
    outfilename = outfilename.sub(/\.html$/, "")
    FileUtils.mkdir_p outfilename
    FileUtils.mv "#{outfilename}.html", outfilename
    FileUtils.mv "#{outfilename}_images", outfilename, force: true
  end
end