class Fbe::Iterate

License
MIT
Copyright
Copyright © 2024-2025 Zerocracy
Author

Yegor Bugayenko (yegor256@gmail.com)
end
pr_number # Return next PR number to process
fetch_and_store_pr(repo_id, pr_number)
# Process pull request
iterator.over(timeout: 600) do |repo_id, pr_number|
iterator.quota_aware
iterator.repeats(10)
iterator.by(‘(and (eq what “pull_request”) (gt number $before))’)
iterator.as(‘pull-requests’)
iterator = Fbe::Iterate.new(fb: fb, loog: loog, options: options, global: global)
@example Processing pull requests with state management
resuming after interruptions.
for that repository. Progress is persisted in the factbase to support
result as context. If the query returns nil, it restarts from the beginning
The iterator executes a query for each repository, passing the previous
- Timeout controls for long-running operations
- Configurable repeat counts per repository
- GitHub API quota awareness to prevent rate limit issues
- Stateful iteration with automatic restart capability
“marker” facts in the factbase and supports features like:
queries while maintaining state between iterations. It tracks progress using
This class provides a DSL for iterating through repositories and executing
Repository iterator with stateful query execution.

def as(label)

Other tags:
    Example: Set label for issue processing -

Raises:
  • (RuntimeError) - If label is already set or nil

Returns:
  • (nil) - Nothing is returned

Parameters:
  • label (String) -- Unique identifier for this iteration type
def as(label)
  raise 'Label is already set' unless @label.nil?
  raise 'Cannot set "label" to nil' if label.nil?
  @label = label
end

def by(query)

Other tags:
    Example: Query for issues after a certain ID -

Raises:
  • (RuntimeError) - If query is already set or nil

Returns:
  • (nil) - Nothing is returned

Parameters:
  • query (String) -- The Factbase query to execute
def by(query)
  raise 'Query is already set' unless @query.nil?
  raise 'Cannot set query to nil' if query.nil?
  @query = query
end

def initialize(fb:, loog:, options:, global:)

Parameters:
  • global (Hash) -- The hash for global caching of API responses
  • options (Judges::Options) -- The options containing repository configuration
  • loog (Loog) -- The logging facility for debug output
  • fb (Factbase) -- The factbase for storing iteration state
def initialize(fb:, loog:, options:, global:)
  @fb = fb
  @loog = loog
  @options = options
  @global = global
  @label = nil
  @since = 0
  @query = nil
  @repeats = 1
  @quota_aware = false
end

def over(timeout: 2 * 60, &)

Other tags:
    Example: Process issues incrementally -

Raises:
  • (RuntimeError) - If block doesn't return an Integer

Returns:
  • (nil) - Nothing is returned

Other tags:
    Yieldreturn: - The value to store as "latest" for next iteration

Other tags:
    Yield: - Repository ID and the result from query execution

Parameters:
  • timeout (Float) -- Maximum seconds to run (default: 120)
def over(timeout: 2 * 60, &)
  raise 'Use "as" first' if @label.nil?
  raise 'Use "by" first' if @query.nil?
  seen = {}
  oct = Fbe.octo(loog: @loog, options: @options, global: @global)
  if oct.off_quota?
    @loog.debug('We are off GitHub quota, cannot even start, sorry')
    return
  end
  repos = Fbe.unmask_repos(loog: @loog, options: @options, global: @global)
  restarted = []
  start = Time.now
  loop do
    if oct.off_quota?
      @loog.info("We are off GitHub quota, time to stop after #{start.ago}")
      break
    end
    repos.each do |repo|
      if oct.off_quota?
        @loog.debug("We are off GitHub quota, we must skip #{repo}")
        break
      end
      if Time.now - start > timeout
        @loog.info("We are doing this for #{start.ago} already, won't check #{repo}")
        next
      end
      next if restarted.include?(repo)
      seen[repo] = 0 if seen[repo].nil?
      if seen[repo] >= @repeats
        @loog.debug("We've seen too many (#{seen[repo]}) in #{repo}, let's see next one")
        next
      end
      rid = oct.repo_id_by_name(repo)
      before = @fb.query(
        "(agg (and (eq what '#{@label}') (eq where 'github') (eq repository #{rid})) (first latest))"
      ).one
      @fb.query("(and (eq what '#{@label}') (eq where 'github') (eq repository #{rid}))").delete!
      before = before.nil? ? @since : before.first
      nxt = @fb.query(@query).one(@fb, before:, repository: rid)
      after =
        if nxt.nil?
          @loog.debug("Next element after ##{before} not suggested, re-starting from ##{@since}: #{@query}")
          restarted << repo
          @since
        else
          @loog.debug("Next is ##{nxt}, starting from it...")
          yield(rid, nxt)
        end
      raise "Iterator must return an Integer, while #{after.class} returned" unless after.is_a?(Integer)
      f = @fb.insert
      f.where = 'github'
      f.repository = rid
      f.latest =
        if after.nil?
          @loog.debug("After is nil at #{repo}, setting the 'latest' to ##{nxt}")
          nxt
        else
          @loog.debug("After is ##{after} at #{repo}, setting the 'latest' to it")
          after
        end
      f.what = @label
      seen[repo] += 1
    end
    unless seen.any? { |r, v| v < @repeats && !restarted.include?(r) }
      @loog.debug("No more repos to scan (out of #{repos.size}), quitting after #{start.ago}")
      break
    end
    if restarted.size == repos.size
      @loog.debug("All #{repos.size} repos restarted, quitting after #{start.ago}")
      break
    end
    if Time.now - start > timeout
      @loog.info("We are iterating for #{start.ago} already, time to give up")
      break
    end
  end
  @loog.debug("Finished scanning #{repos.size} repos in #{start.ago}: #{seen.map { |k, v| "#{k}:#{v}" }.join(', ')}")
end

def quota_aware

Other tags:
    Example: Enable quota awareness -

Returns:
  • (nil) - Nothing is returned
def quota_aware
  @quota_aware = true
end

def repeats(repeats)

Other tags:
    Example: Process up to 100 items per repository -

Raises:
  • (RuntimeError) - If repeats is nil or not positive

Returns:
  • (nil) - Nothing is returned

Parameters:
  • repeats (Integer) -- The maximum iterations per repository
def repeats(repeats)
  raise 'Cannot set "repeats" to nil' if repeats.nil?
  raise 'The "repeats" must be a positive integer' unless repeats.positive?
  @repeats = repeats
end