class Fbe::Iterate
- License
- MIT
Copyright - Copyright © 2024-2025 Zerocracy
Author -
Yegor Bugayenko (yegor256@gmail.com)
end
pr_number # Return next PR number to process
fetch_and_store_pr(repo_id, pr_number)
# Process pull request
iterator.over(timeout: 600) do |repo_id, pr_number|
iterator.quota_aware
iterator.repeats(10)
iterator.by(‘(and (eq what “pull_request”) (gt number $before))’)
iterator.as(‘pull-requests’)
iterator = Fbe::Iterate.new(fb: fb, loog: loog, options: options, global: global)
@example Processing pull requests with state management
resuming after interruptions.
for that repository. Progress is persisted in the factbase to support
result as context. If the query returns nil, it restarts from the beginning
The iterator executes a query for each repository, passing the previous
- Timeout controls for long-running operations
- Configurable repeat counts per repository
- GitHub API quota awareness to prevent rate limit issues
- Stateful iteration with automatic restart capability
“marker” facts in the factbase and supports features like:
queries while maintaining state between iterations. It tracks progress using
This class provides a DSL for iterating through repositories and executing
Repository iterator with stateful query execution.
- Copyright © 2024-2025 Zerocracy
- MIT
def as(label)
- Example: Set label for issue processing -
Raises:
-
(RuntimeError)
- If label is already set or nil
Returns:
-
(nil)
- Nothing is returned
Parameters:
-
label
(String
) -- Unique identifier for this iteration type
def as(label) raise 'Label is already set' unless @label.nil? raise 'Cannot set "label" to nil' if label.nil? @label = label end
def by(query)
- Example: Query for issues after a certain ID -
Raises:
-
(RuntimeError)
- If query is already set or nil
Returns:
-
(nil)
- Nothing is returned
Parameters:
-
query
(String
) -- The Factbase query to execute
def by(query) raise 'Query is already set' unless @query.nil? raise 'Cannot set query to nil' if query.nil? @query = query end
def initialize(fb:, loog:, options:, global:)
-
global
(Hash
) -- The hash for global caching of API responses -
options
(Judges::Options
) -- The options containing repository configuration -
loog
(Loog
) -- The logging facility for debug output -
fb
(Factbase
) -- The factbase for storing iteration state
def initialize(fb:, loog:, options:, global:) @fb = fb @loog = loog @options = options @global = global @label = nil @since = 0 @query = nil @repeats = 1 @quota_aware = false end
def over(timeout: 2 * 60, &)
- Example: Process issues incrementally -
Raises:
-
(RuntimeError)
- If block doesn't return an Integer
Returns:
-
(nil)
- Nothing is returned
Other tags:
- Yieldreturn: - The value to store as "latest" for next iteration
Other tags:
- Yield: - Repository ID and the result from query execution
Parameters:
-
timeout
(Float
) -- Maximum seconds to run (default: 120)
def over(timeout: 2 * 60, &) raise 'Use "as" first' if @label.nil? raise 'Use "by" first' if @query.nil? seen = {} oct = Fbe.octo(loog: @loog, options: @options, global: @global) if oct.off_quota? @loog.debug('We are off GitHub quota, cannot even start, sorry') return end repos = Fbe.unmask_repos(loog: @loog, options: @options, global: @global) restarted = [] start = Time.now loop do if oct.off_quota? @loog.info("We are off GitHub quota, time to stop after #{start.ago}") break end repos.each do |repo| if oct.off_quota? @loog.debug("We are off GitHub quota, we must skip #{repo}") break end if Time.now - start > timeout @loog.info("We are doing this for #{start.ago} already, won't check #{repo}") next end next if restarted.include?(repo) seen[repo] = 0 if seen[repo].nil? if seen[repo] >= @repeats @loog.debug("We've seen too many (#{seen[repo]}) in #{repo}, let's see next one") next end rid = oct.repo_id_by_name(repo) before = @fb.query( "(agg (and (eq what '#{@label}') (eq where 'github') (eq repository #{rid})) (first latest))" ).one @fb.query("(and (eq what '#{@label}') (eq where 'github') (eq repository #{rid}))").delete! before = before.nil? ? @since : before.first nxt = @fb.query(@query).one(@fb, before:, repository: rid) after = if nxt.nil? @loog.debug("Next element after ##{before} not suggested, re-starting from ##{@since}: #{@query}") restarted << repo @since else @loog.debug("Next is ##{nxt}, starting from it...") yield(rid, nxt) end raise "Iterator must return an Integer, while #{after.class} returned" unless after.is_a?(Integer) f = @fb.insert f.where = 'github' f.repository = rid f.latest = if after.nil? @loog.debug("After is nil at #{repo}, setting the 'latest' to ##{nxt}") nxt else @loog.debug("After is ##{after} at #{repo}, setting the 'latest' to it") after end f.what = @label seen[repo] += 1 end unless seen.any? { |r, v| v < @repeats && !restarted.include?(r) } @loog.debug("No more repos to scan (out of #{repos.size}), quitting after #{start.ago}") break end if restarted.size == repos.size @loog.debug("All #{repos.size} repos restarted, quitting after #{start.ago}") break end if Time.now - start > timeout @loog.info("We are iterating for #{start.ago} already, time to give up") break end end @loog.debug("Finished scanning #{repos.size} repos in #{start.ago}: #{seen.map { |k, v| "#{k}:#{v}" }.join(', ')}") end
def quota_aware
- Example: Enable quota awareness -
Returns:
-
(nil)
- Nothing is returned
def quota_aware @quota_aware = true end
def repeats(repeats)
- Example: Process up to 100 items per repository -
Raises:
-
(RuntimeError)
- If repeats is nil or not positive
Returns:
-
(nil)
- Nothing is returned
Parameters:
-
repeats
(Integer
) -- The maximum iterations per repository
def repeats(repeats) raise 'Cannot set "repeats" to nil' if repeats.nil? raise 'The "repeats" must be a positive integer' unless repeats.positive? @repeats = repeats end