Top Level Namespace

Defined Under Namespace

Modules: ForceArray, Jekyll, LiquiDocFilters, Liquidoc Classes: Array, AsciiDocument, Build, BuildConfig, BuildConfigStep, DataFiles, DataObj, DataSrc, Hash, String

Instance Method Summary

• #algolia_index_cmd(build, apikey = nil, args) ⇒ Object
• #asciidocify(doc, build) ⇒ Object
• #cli_liquify(data_files = nil, template_file = nil, output_file = nil, passed_vars) ⇒ Object
• #config_build(config_file, config_vars = {}, data_files = nil, parse = false) ⇒ Object

  Establish source, template, index, etc details for build jobs from a config file.

• #copy_assets(src, dest, inclusive = true, missing = 'exit') ⇒ Object

  Copy images and other files into target dir.

• #derive_backend(type, out_file) ⇒ Object
• #execute_command(cmd) ⇒ Object

  Execute ===.

• #explainer_init(out = nil) ⇒ Object
• #generate_file(content, target) ⇒ Object
• #generate_site(doc, build) ⇒ Object
• #get_payload(data_files) ⇒ Object

  Action-specific procs === PARSE-type build procs ===.

• #ingest_attributes(attr_file) ⇒ Object

  Gather attributes from one or more fixed attributes files.

• #ingest_data(datasrc) ⇒ Object

  Pull in a semi-structured data file, converting contents to a Ruby object.

• #iterate_build(cfg) ⇒ Object
• #jekyll_serve(build) ⇒ Object

  DEPLOY procs ===.

• #liquify(data_obj, template_file, output = "stdout") ⇒ Object

  Parse given data using given template, generating given output.

• #load_jekyll_data(build) ⇒ Object
• #parse_regex(data_file, pattern) ⇒ Object
• #regurgidata(data_obj, output) ⇒ Object
• #render_doc(doc, build) ⇒ Object
• #validate_file_input(file, type) ⇒ Object

  Verify files exist.

Instance Method Details

#algolia_index_cmd(build, apikey = nil, args) ⇒ Object

# File 'lib/liquidoc.rb', line 1154

def algolia_index_cmd build, apikey=nil, args
  unless build.search and build.search['index']
    @logger.warn "No index configuration found for build; jekyll-algolia operation skipped for this build."
    return false
  else
    unless apikey
      @logger.warn "No Algolia admin API key passed; skipping jekyll-algolia operation for this build."
      return false
    else
      return "ALGOLIA_INDEX_NAME='#{build.search['index']}' ALGOLIA_API_KEY='#{apikey}' bundle exec jekyll algolia #{@search_index_dry} #{args} "
    end
  end
end

#asciidocify(doc, build) ⇒ Object

# File 'lib/liquidoc.rb', line 1041

def asciidocify doc, build
  @logger.debug "Executing Asciidoctor render operation for #{build.output}."
  to_file = build.output
  unless doc.type == build.doctype
    if build.doctype.nil? # set a default doctype equal to our LiquiDoc action doc type
      build.set("doctype", doc.type)
    end
  end
  # unfortunately we have to treat attributes accumilation differently for Jekyll vs Asciidoctor
  attrs = doc.attributes # Start with attributes added at the action level; no more writing to doc obj
  # Handle properties files array as attributes files and
  # add the ingested attributes to local var
  begin
    if build.prop_files_array
      ingested = ingest_attributes(build.prop_files_array)
      attrs.merge!(ingested)
    else
      puts build.prop_files_array
    end
  rescue Exception => ex
    @logger.warn "Attributes failed to merge. #{ex}" # Shd only trigger if build.props exists
    raise
  end
  if build.backend == "html5" # Insert a stylesheet
    attrs.merge!({"stylesheet"=>build.style}) if build.style
  end
  # Add attributes from config file build section
  attrs.merge!(build.attributes) # Finally merge attributes from the build step
  # Add attributes from command-line -a args
  @logger.debug "Final pre-Asciidoctor attributes: #{attrs.to_yaml}"
  # Perform the aciidoctor convert
  if build.backend == "pdf"
    @logger.info "Generating PDF. This can take some time..."
    attrs.merge!({"pdf-theme"=>build.style}) if build.style
  end
  Asciidoctor.convert_file(
    doc.index,
    to_file: to_file,
    attributes: attrs,
    require: "pdf",
    backend: build.backend,
    doctype: build.doctype,
    safe: "unsafe",
    sourcemap: true,
    verbose: @verbose,
    mkdirs: true,
  )
  @logger.info "Rendered file #{to_file}."
end

#cli_liquify(data_files = nil, template_file = nil, output_file = nil, passed_vars) ⇒ Object

# File 'lib/liquidoc.rb', line 889

def cli_liquify data_files=nil, template_file=nil, output_file=nil, passed_vars
  # converts command-line options into liquify or regurgidata inputs
  data_obj = DataObj.new()
  if output_file
    output = output_file
  else
    output = "stdout"
  end
  if data_files
    payload = get_payload(data_files)
    data_obj.add_payload!(payload)
  end
  if template_file
    # data_obj.add_data!(ingested, "data") if df
    data_obj.add_data!(passed_vars, "vars") if passed_vars
    liquify(data_obj, template_file, output)
  else
    data_obj.remove_scope("vars")
    data_obj.add_data!(passed_vars) if passed_vars
    regurgidata(data_obj, output)
  end
end

#config_build(config_file, config_vars = {}, data_files = nil, parse = false) ⇒ Object

Establish source, template, index, etc details for build jobs from a config file

# File 'lib/liquidoc.rb', line 84

def config_build config_file, config_vars={}, data_files=nil, parse=false
  @logger.debug "Using config file #{config_file}."
  validate_file_input(config_file, "config")
  config_base = File.read(config_file)
  if config_vars.length > 0 or data_files or parse or config_base.contains_liquid?
    @logger.debug "Config_vars: #{config_vars.length}"
  # If config variables are passed on the CLI, we want to parse the config file
  # and use the parsed version for the rest fo this routine
    config_out = "#{@build_dir}/pre/#{File.basename(config_file)}"
    data_obj = DataObj.new()
    if data_files
      payload = get_payload(data_files)
      data_obj.add_payload!(payload)
    end
    data_obj.add_data!(config_vars, "vars")
    liquify(data_obj, config_file, config_out)
    config_file = config_out
    @logger.debug "Config parsed! Using #{config_out} for build."
  end
  validate_file_input(config_file, "config")
  begin
    config = YAML.load_file(config_file)
  rescue Exception => ex
    unless File.exists?(config_file)
      @logger.error "Config file #{config_file} not found."
    else
      @logger.error "Problem loading config file #{config_file}. #{ex} Exiting."
    end
    raise "ConfigFileError"
  end
  # TESTS
  # puts config[0].argify
  cfg = BuildConfig.new(config) # convert the config file to a new object called 'cfg'
  if @safemode
    commands = ""
    cfg.steps.each do |step|
      if step['action'] == "execute"
        commands = commands + "> " + step['command'] + "\n"
      end
    end
    unless commands.to_s.strip.empty?
      puts "\nWARNING: This routine will execute the following shell commands:\n\n#{commands}"
      ui = HighLine.new
      answer = ui.ask("\nDo you approve? (YES/no): ")
      raise "CommandExecutionsNotAuthorized" unless answer.strip == "YES"
    end
  end
  iterate_build(cfg)
end

#copy_assets(src, dest, inclusive = true, missing = 'exit') ⇒ Object

Copy images and other files into target dir

# File 'lib/liquidoc.rb', line 941

def copy_assets src, dest, inclusive=true, missing='exit'
  unless File.file?(src)
    unless inclusive then src = src + "/." end
  end
  src_to_dest = "#{src} to #{dest}"
  unless (File.file?(src) || File.directory?(src))
    case missing
    when "warn"
      @logger.warn "Skipping migrate action (#{src_to_dest}); source not found."
      return
    when "skip"
      @logger.debug "Skipping migrate action (#{src_to_dest}); source not found."
      return
    when "exit"
      @logger.error "Unexpected missing source in migrate action (#{src_to_dest})."
      raise "MissingSourceExit"
    end
  end
  @logger.debug "Copying #{src_to_dest}"
  begin
    FileUtils.mkdir_p(dest) unless File.directory?(dest)
    if File.directory?(src)
      FileUtils.cp_r(src, dest)
    else
      FileUtils.cp(src, dest)
    end
    @logger.info "Copied #{src} to #{dest}."
  rescue Exception => ex
    @logger.error "Problem while copying assets. #{ex.message}"
    raise
  end
end

#derive_backend(type, out_file) ⇒ Object

# File 'lib/liquidoc.rb', line 1018

def derive_backend type, out_file
  case File.extname(out_file)
  when ".pdf"
    backend = "pdf"
  else
    backend = "html5"
  end
  return backend
end

#execute_command(cmd) ⇒ Object

Execute

# File 'lib/liquidoc.rb', line 1172

def execute_command cmd
  stdout, stderr, status = Open3.capture3(cmd.command)
  failed = true if status.to_s.include?("exit 1")
  unless cmd.options
    puts stdout
    puts stderr if failed
  else
    if failed && cmd.options['error']
      @logger.warn cmd.options['error']['message'] if cmd.options['error']['message']
      if cmd.options['error']['response'] == "exit"
        @logger.error "Command failure: #{stderr}"
        raise "CommandExecutionException"
      end
    end
    if cmd.options['outfile']
      contents = stdout
      if cmd.options['outfile']
        contents = "#{cmd.options['outfile']['prepend']}\n#{stdout}" if cmd.options['outfile']['prepend']
        contents = "#{stdout}\n#{cmd.options['outfile']['append']}" if cmd.options['outfile']['append']
        generate_file(contents, cmd.options['outfile']['path'])
      end
      if cmd.options['stdout']
        puts stdout
      end
    end
  end
end

#explainer_init(out = nil) ⇒ Object

# File 'lib/liquidoc.rb', line 224

def explainer_init out=nil
  unless @explainer
    if out == "STDOUT"
      @explainer = Logger.new(STDOUT)
    else
      out = "#{@build_dir}/pre/config-explainer.adoc" if out.nil?
      File.open(out, 'w') unless File.exists?(out)
      file = File.open(out, File::WRONLY)
      begin
        @explainer = Logger.new(file)
      rescue Exception => ex
        @logger.error ex
        raise "ExplainerCreateError"
      end
    end
    @explainer.formatter = proc do |severity, datetime, progname, msg|
      "#{msg}\n"
    end
  end
end

#generate_file(content, target) ⇒ Object

# File 'lib/liquidoc.rb', line 245

def generate_file content, target
  base_path = File.dirname(target)
  begin
    FileUtils::mkdir_p(base_path) unless File.exists?(base_path)
    File.open(target, 'w') { |file| file.write(content) } # saves file
  rescue Exception => ex
    @logger.error "Failed to save output.\n#{ex.class} #{ex.message}"
    raise "FileNotBuilt"
  end
  if File.exists?(target)
    @logger.info "File built: #{target}"
  else
    @logger.error "Hrmp! File not built."
    raise "FileNotBuilt"
  end
end

#generate_site(doc, build) ⇒ Object

# File 'lib/liquidoc.rb', line 1091

def generate_site doc, build
  case build.backend
  when "jekyll"
    attrs = doc.attributes
    build.add_config_file("_config.yml") unless build.prop_files_array
    jekyll = load_jekyll_data(build) # load the first Jekyll config file locally
    attrs.merge! ({"base_dir" => jekyll['source']}) # Sets default Asciidoctor base_dir to == Jekyll root
    # write all AsciiDoc attributes to a config file for Jekyll to ingest
    attrs.merge!(build.attributes) if build.attributes
    attrs = {"asciidoctor" => {"attributes" => attrs} }
    attrs_yaml = attrs.to_yaml # Convert it all back to Yaml, as we're going to write a file to feed back to Jekyll
    File.open("#{@build_dir}/pre/attributes_#{@render_count}.yml", 'w') { |file| file.write(attrs_yaml) }
    build.add_config_file("#{@build_dir}/pre/attributes_#{@render_count}.yml")
    config_list = build.prop_files_array.join(',') # flatten the Array back down for the CLI
    quiet = "--quiet" if @quiet || @explicit
    if build.props['arguments']
      opts_args_file = "#{@build_dir}/pre/jekyll_opts_args_#{@render_count}.yml"
      opts_args = build.props['arguments']
      File.open(opts_args_file, 'w') { |file|
      file.write(opts_args.to_yaml)}
      config_list << ",#{opts_args_file}"
    end
    base_args = "--config #{config_list}"
    base_args += " --trace" if @verbose
    command = "bundle exec jekyll build #{base_args} #{quiet}"
    if @search_index
      # TODO enable config-based admin api key ingest once config is dynamic
      command = algolia_index_cmd(build, @search_api_key, base_args)
      @logger.warn "Search indexing failed." unless command
    end
  end
  if command
    @logger.info "Running #{command}"
    @logger.debug "Final pre-jekyll-asciidoc attributes: #{doc.attributes.to_yaml} "
    system command
  end
  jekyll_serve(build) if @jekyll_serve
end

#get_payload(data_files) ⇒ Object

Action-specific procs

PARSE-type build procs

# File 'lib/liquidoc.rb', line 770

def get_payload data_files
  # data_files: a proper DataFile object
  payload = []
  data_files.sources.each do |src|
    obj = {}
    begin
      data = ingest_data(src) # Extract data from file
    rescue Exception => ex
      @logger.error "#{ex.class}: #{ex.message}"
      raise "DataFileReadFail (#{src.file})"
    end
    obj['name'] = src.name
    obj['data'] = data
    payload << obj
  end
  return payload
end

#ingest_attributes(attr_file) ⇒ Object

Gather attributes from one or more fixed attributes files

# File 'lib/liquidoc.rb', line 979

def ingest_attributes attr_file
  attr_files_array = attr_file.force_array
  attrs = {}
  attr_files_array.each do |f|
    if f.include? ":"
      file = f.split(":")
      filename = file[0]
      block_name = file[1]
    else
      filename = f
      block_name = false
    end
    validate_file_input(filename, "attributes")
    begin
      new_attrs = YAML.load_file(filename)
      if block_name
        begin
          new_attrs = new_attrs[block_name]
        rescue
          raise "InvalidAttributesBlock (#{filename}:#{block_name})"
        end
      end
    rescue Exception => ex
      @logger.error "Attributes block invalid. #{ex.class}: #{ex.message}"
      raise "AttributeBlockError"
    end
    begin
      if new_attrs.is_a? Hash
        attrs.merge!new_attrs
      else
        @logger.warn "The AsciiDoc attributes file #{filename} is not formatted as a hash, so its data was not ingested."
      end
    rescue Exception => ex
      raise "AttributesMergeError #{ex.message}"
    end
  end
  return attrs
end

#ingest_data(datasrc) ⇒ Object

Pull in a semi-structured data file, converting contents to a Ruby object

# File 'lib/liquidoc.rb', line 789

def ingest_data datasrc
  raise "InvalidDataSrcObject" unless datasrc.is_a? DataSrc
  case datasrc.type
  when "yml"
    begin
      data = YAML.load_file(datasrc.file)
    rescue Exception => ex
      @logger.error "There was a problem with the data file. #{ex.message}"
    end
  when "json"
    begin
      data = JSON.parse(File.read(datasrc.file))
    rescue Exception => ex
      @logger.error "There was a problem with the data file. #{ex.message}"
    end
  when "xml"
    begin
      data = Crack::XML.parse(File.read(datasrc.file))
      data = data['root']
    rescue Exception => ex
      @logger.error "There was a problem with the data file. #{ex.message}"
    end
  when "csv"
    data = []
    i = 0
    begin
      CSV.foreach(datasrc.file, headers: true, skip_blanks: true) do |row|
        data[i] = row.to_hash
        i = i+1
      end
    rescue
      @logger.error "The CSV format is invalid."
    end
  when "regex"
    if datasrc.pattern
      data = parse_regex(datasrc.file, datasrc.pattern)
    else
      @logger.error "You must supply a regex pattern with your free-form data file."
      raise "MissingRegexPattern"
    end
  when "adoc"
    begin
      doc = Asciidoctor.load_file(datasrc.file)
      data = doc.attributes
    rescue
      @logger.error "Problem with AsciiDoc source file. Attributes not ingested."
    end
  end
  return data
end

#iterate_build(cfg) ⇒ Object

# File 'lib/liquidoc.rb', line 134

def iterate_build cfg
  stepcount = 0
  for step in cfg.steps # iterate through each node in the 'config' object, which should start with an 'action' parameter
    stepcount = stepcount + 1
    step = BuildConfigStep.new(step) # create an instance of the Action class, validating the top-level step hash (now called 'step') in the process
    @explainer.info step.message
    type = step.type
    case type # a switch to evaluate the 'action' parameter for each step in the iteration...
    when "parse"
      builds = step.builds
      data_obj = DataObj.new()
      if step.data
        data_files = DataFiles.new(step.data)
        payload = get_payload(data_files)
        data_obj.add_payload!(payload)
      end
      builds.each do |bld|
        build = Build.new(bld, type, data_obj) # create an instance of the Build class; Build.new accepts a 'bld' hash & action 'type'
        if build.template
          # Prep & perform a Liquid-parsed build
          @explainer.info build.message
          build.add_data!(build.variables, "vars") if build.variables
          includes_dirs = @includes_dirs
          includes_dirs = build.includes_dirs if build.includes_dirs
          build.add_data!({:includes_dirs=>includes_dirs})
          liquify(build.data, build.template, build.output) # perform the liquify operation
        else # Prep & perform a direct conversion
          # Delete nested data and vars objects
          build.data.remove_scope("data")
          build.data.remove_scope("vars")
          # Add vars from CLI or config args
          build.data.add_data!(build.variables) unless build.variables.empty?
          regurgidata(build.data, build.output)
        end
      end
    when "migrate"
      inclusive = true
      missing = "exit"
      if step.options
        inclusive = step.options['inclusive'] if step.options.has_key?("inclusive")
        missing = step.options['missing'] if step.options.has_key?("missing")
      end
      copy_assets(step.source, step.target, inclusive, missing)
    when "render"
      validate_file_input(step.source, "source") if step.source
      builds = step.builds
      for bld in builds
        doc = AsciiDocument.new(step.source)
        if step.data
          attrs = ingest_attributes(step.data)
          doc.add_attrs!(attrs) # Set attributes from the action-level data file
        end
        build = Build.new(bld, type) # create an instance of the Build class; Build.new accepts a 'bld' hash & action 'type' string
        build.set("backend", derive_backend(doc.type, build.output) ) unless build.backend
        @explainer.info build.message
        render_doc(doc, build) # perform the render operation
      end
    when "deploy"
      @logger.warn "Deploy actions are limited and experimental."
      jekyll_serve(build)
    when "execute"
      @logger.info "Executing shell command: #{step.command}"
      execute_command(step)
    else
      @logger.warn "The action `#{type}` is not valid."
    end
  end
end

#jekyll_serve(build) ⇒ Object

DEPLOY procs

# File 'lib/liquidoc.rb', line 1143

def jekyll_serve build
  # Locally serve Jekyll as per the primary Jekyll config file
  @logger.debug "Attempting Jekyll serve operation."
  config_file = build.props['files'][0]
  if build.props['arguments']
    opts_args = build.props['arguments'].argify
  end
  command = "bundle exec jekyll serve --config #{config_file} #{opts_args} --no-watch --skip-initial-build"
  system command
end

#liquify(data_obj, template_file, output = "stdout") ⇒ Object

Parse given data using given template, generating given output

# File 'lib/liquidoc.rb', line 867

def liquify data_obj, template_file, output="stdout"
  validate_file_input(template_file, "template")
  # inject :includes_dirs as needed
  data_obj.add_data!({'includes_dirs' => @includes_dirs}) unless data_obj.data['includes_dirs']
  begin
    template = File.read(template_file) # reads the template file
    template = Liquid::Template.parse(template) # compiles template
    rendered = template.render(data_obj.data) # renders the output
  rescue Exception => ex
    message = "Problem rendering Liquid template. #{template_file}\n" \
      "#{ex.class} thrown. #{ex.message}"
    @logger.error message
    raise message
  end
  unless output == "stdout" || @output_type == "stdout"
    output_file = output
    generate_file(rendered, output_file)
  else # if stdout
    puts rendered
  end
end

#load_jekyll_data(build) ⇒ Object

# File 'lib/liquidoc.rb', line 1130

def load_jekyll_data build
  data = {}
  build.prop_files_array.each do |file|
    settings = YAML.load_file(file)
    data.merge!settings if settings
  end
  return data
end

#parse_regex(data_file, pattern) ⇒ Object

# File 'lib/liquidoc.rb', line 840

def parse_regex data_file, pattern
  records = []
  pattern_re = /#{pattern}/
  @logger.debug "Using regular expression #{pattern} to parse data file."
  groups = pattern_re.names
  begin
    File.open(data_file, "r") do |file_proc|
      file_proc.each_line do |row|
        matches = row.match(pattern_re)
        if matches
          row_h = {}
          groups.each do |var| # loop over the named groups, adding their key & value to the row_h hash
            row_h.merge!(var => matches[var])
          end
          records << row_h # add the row to the records array
        end
      end
    end
    output = records
  rescue Exception => ex
    @logger.error "Something went wrong trying to parse the free-form file. #{ex.class} thrown. #{ex.message}"
    raise "Freeform parse error"
  end
  return output
end

#regurgidata(data_obj, output) ⇒ Object

# File 'lib/liquidoc.rb', line 912

def regurgidata data_obj, output
  # converts data object from one format directly to another
  raise "UnrecognizedFileExtension" unless File.extname(output).match(/\.yml|\.json|\.xml|\.csv/)
  case File.extname(output)
    when ".yml"
      new_data = data_obj.data.to_yaml
    when ".json"
      new_data = data_obj.data.to_json
    when ".xml"
      @logger.warn "XML output not yet implemented."
    when ".csv"
      @logger.warn "CSV output not yet implemented."
  end
  if new_data
    begin
      generate_file(new_data, output)
      @logger.info "Data converted and saved to #{output}."
    rescue Exception => ex
      @logger.error "#{ex.class}: #{ex.message}"
      raise "FileWriteError"
    end
  end
end

#render_doc(doc, build) ⇒ Object

# File 'lib/liquidoc.rb', line 1028

def render_doc doc, build
  @render_count += 1
  @logger.info "### Build ##{@render_count}"
  case build.backend
  when "html5", "pdf"
    asciidocify(doc, build)
  when "jekyll"
    generate_site(doc, build)
  else
    raise "UnrecognizedBackend"
  end
end

#validate_file_input(file, type) ⇒ Object

Verify files exist

# File 'lib/liquidoc.rb', line 208

def validate_file_input file, type
  @logger.debug "Validating input file for #{type} file #{file}"
  error = false
  unless file.is_a?(String) and !file.nil?
    error = "The #{type} filename (#{file}) is not valid."
  else
    unless File.exists?(file)
      error = "The #{type} file (#{file}) was not found."
    end
  end
  if error
    @logger.error "Could not validate input file: #{error}"
    raise "InvalidInput"
  end
end