# encoding: utf-8 module SportDb class Reader include LogUtils::Logging ## make models available in sportdb module by default with namespace # e.g. lets you use Team instead of Models::Team include SportDB::Models attr_reader :include_path def initialize( include_path, opts={}) @include_path = include_path end def load_setup( setup ) ary = load_fixture_setup( setup ) load( ary ) end # method load_setup ## fix/todo: rename ?? def load_fixture_setup( name ) ## todo/fix: cleanup quick and dirty code path = "#{include_path}/#{name}.yml" logger.info "parsing data '#{name}' (#{path})..." text = File.read_utf8( path ) hash = YAML.load( text ) ### build up array for fixtures from hash ary = [] hash.each do |key_wild, value_wild| key = key_wild.to_s.strip logger.debug "yaml key:#{key_wild.class.name} >>#{key}<<, value:#{value_wild.class.name} >>#{value_wild}<<" if value_wild.kind_of?( String ) # assume non-event data ary << value_wild elsif value_wild.kind_of?( Array ) # assume non_event data as array of strings ary = ary + value_wild elsif value_wild.kind_of?( Hash ) # assume event data value_wild.each do |event_key, event_values| # e.g. # at.2012/13: at/2012_13/bl, at/2012_13/bl2 # becomes # [ 'at.2012/13', 'at/2012_13/bl', 'at/2012_13/bl2' ] ary << ( [ event_key.to_s ] + event_values.split(',') ) end else logger.error "unknow fixture type in setup (yaml key:#{key_wild.class.name} >>#{key}<<, value:#{value_wild.class.name} >>#{value_wild}<<); skipping" end end logger.debug "fixture setup:" logger.debug ary.to_json ary end # load_fixture_setup def load( ary ) # convenience helper for all-in-one reader logger.debug "enter load(include_path=>>#{include_path}<<):" logger.debug ary.to_json ary.each do |rec| if rec.kind_of?( String ) ## assume single fixture name name = rec if name =~ /^circuits/ # e.g. circuits.txt in formula1.db load_tracks( name ) elsif name =~ /^drivers/ # e.g. drivers.txt in formula1.db load_persons( name ) elsif name =~ /^teams/ # e.g. teams.txt in formula1.db load_teams( name ) elsif name =~ /\/races/ # e.g. 2013/races.txt in formula1.db load_races( name ) elsif name =~ /\/squads/ || name =~ /\/rosters/ # e.g. 2013/squads.txt in formula1.db load_rosters( name ) elsif name =~ /\/([0-9]{2})-/ load_records( name ) # e.g. 2013/04-gp-monaco.txt in formula1.db elsif name =~ /^seasons/ load_seasons( name ) elsif name =~ /^leagues/ if name =~ /club/ # e.g. leagues_club load_leagues( name, club: true ) else # e.g. leagues load_leagues( name ) end elsif name =~ /^([a-z]{2})\/leagues/ # auto-add country code (from folder structure) for country-specific leagues # e.g. at/leagues country_key = $1 country = Country.find_by_key!( country_key ) load_leagues( name, club: true, country_id: country.id ) elsif name =~ /^([a-z]{2})\/teams/ # auto-add country code (from folder structure) for country-specific teams # e.g. at/teams at/teams.2 de/teams etc. country_key = $1 country = Country.find_by_key!( country_key ) load_teams( name, club: true, country_id: country.id ) elsif name =~ /\/teams/ if name =~ /club/ # club teams (many countries) # e.g. club/europe/teams load_teams( name, club: true ) else # assume national teams # e.g. world/teams amercia/teams_n load_teams( name, national: true ) end else logger.error "unknown sportdb fixture type >#{name}<" # todo/fix: exit w/ error end else # more than one item in record? assume fixture starting w/ event key # assume first item is key # assume second item is event plus fixture # assume option third,etc are fixtures (e.g. bl2, etc.) event_key = rec[0] # e.g. at.2012/13 event_name = rec[1] # e.g. at/2012_13/bl fixture_names = rec[1..-1] # e.g. at/2012_13/bl, at/2012_13/bl2 load_event( event_name ) fixture_names.each do |fixture_name| load_fixtures( event_key, fixture_name ) end end end # each ary end # method load def load_leagues( name, more_values={} ) path = "#{include_path}/#{name}.txt" logger.info "parsing data '#{name}' (#{path})..." reader = ValuesReader.new( path, more_values ) reader.each_line do |new_attributes, values| League.create_or_update_from_values( new_attributes, values ) end # each lines Prop.create_from_fixture!( name, path ) end # load_leagues def load_tracks( name, more_values={} ) path = "#{include_path}/#{name}.txt" logger.info "parsing data '#{name}' (#{path})..." reader = ValuesReader.new( path, more_values ) reader.each_line do |new_attributes, values| Track.create_or_update_from_values( new_attributes, values ) end # each lines Prop.create_from_fixture!( name, path ) end # load_tracks def load_persons( name, more_values={} ) path = "#{include_path}/#{name}.txt" logger.info "parsing data '#{name}' (#{path})..." reader = ValuesReader.new( path, more_values ) reader.each_line do |new_attributes, values| Person.create_or_update_from_values( new_attributes, values ) end # each lines Prop.create_from_fixture!( name, path ) end # load_persons def load_seasons( name ) path = "#{include_path}/#{name}.yml" logger.info "parsing data '#{name}' (#{path})..." reader = HashReader.new( path ) #### ## fix!!!!! ## use Season.create_or_update_from_hash or similar ## use Season.create_or_update_from_hash_reader?? or similar # move parsing code to model reader.each_typed do |key, value| ## puts "processing event attrib >>#{key}<< >>#{value}<<..." if key == 'seasons' logger.debug "#{value.class.name}: >>#{value}<<" ## nb: assume value is an array value.each do |item| season_attribs = {} season = Season.find_by_key( item.to_s.strip ) ## check if it exists if season.present? logger.debug "update season #{season.id}-#{season.key}:" else logger.debug "create season:" season = Season.new season_attribs[ :key ] = item.to_s.strip end season_attribs[:title] = item.to_s.strip logger.debug season_attribs.to_json season.update_attributes!( season_attribs ) end else logger.error "unknown seasons key; skipping" end end # each key,value Prop.create_from_fixture!( name, path ) end # load_seasons def load_event( name ) path = "#{include_path}/#{name}.yml" logger.info "parsing data '#{name}' (#{path})..." #### ## fix!!!!! ## use Event.create_or_update_from_hash or similar ## use Event.create_or_update_from_hash_reader?? or similar # move parsing code to model reader = HashReader.new( path ) event_attribs = {} reader.each_typed do |key, value| ## puts "processing event attrib >>#{key}<< >>#{value}<<..." if key == 'league' league = League.find_by_key( value.to_s.strip ) ## check if it exists if league.present? event_attribs['league_id'] = league.id else logger.error "league with key >>#{value.to_s.strip}<< missing" exit 1 end elsif key == 'season' season = Season.find_by_key( value.to_s.strip ) ## check if it exists if season.present? event_attribs['season_id'] = season.id else logger.error "season with key >>#{value.to_s.strip}<< missing" exit 1 end elsif key == 'start_at' if value.is_a?(DateTime) || value.is_a?(Date) start_at = value else # assume it's a string start_at = DateTime.strptime( value.to_s.strip, '%Y-%m-%d' ) end event_attribs['start_at'] = start_at elsif key == 'teams' ## assume teams value is an array team_ids = [] value.each do |item| team_key = item.to_s.strip team = Team.find_by_key!( team_key ) team_ids << team.id end event_attribs['team_ids'] = team_ids elsif key == 'team3' ## for now always assume false # todo: fix - use value and convert to boolean if not boolean event_attribs['team3'] = false else ## todo: add a source location struct to_s or similar (file, line, col) logger.error "unknown event attrib; skipping attrib" end end # each key,value event = Event.find_by_league_id_and_season_id( event_attribs['league_id'], event_attribs['season_id']) ## check if it exists if event.present? logger.debug "*** update event #{event.id}-#{event.key}:" else logger.debug "*** create event:" event = Event.new end logger.debug event_attribs.to_json event.update_attributes!( event_attribs ) Prop.create_from_fixture!( name, path ) end # load_event def load_fixtures_from_string( event_key, text ) # load from string (e.g. passed in via web form) ## todo/fix: move code into LineReader e.g. use LineReader.fromString() - why? why not? reader = StringLineReader.new( text ) load_fixtures_worker( event_key, reader ) ## fix add prop ### Prop.create!( key: "db.#{fixture_name_to_prop_key(name)}.version", value: "file.txt.#{File.mtime(path).strftime('%Y.%m.%d')}" ) end def load_fixtures( event_key, name ) # load from file system path = "#{include_path}/#{name}.txt" logger.info "parsing data '#{name}' (#{path})..." SportDB.lang.lang = LangChecker.new.analyze( name, include_path ) reader = LineReader.new( path ) load_fixtures_worker( event_key, reader ) Prop.create_from_fixture!( name, path ) end def load_records( name ) path = "#{include_path}/#{name}.txt" logger.info "parsing data '#{name}' (#{path})..." ### SportDB.lang.lang = LangChecker.new.analyze( name, include_path ) reader = LineReader.new( path ) ## for now: use all tracks (later filter/scope by event) # @known_tracks = Track.known_tracks_table ## fix: add @known_teams - for now; use teams (not scoped by event) load_records_worker( reader ) Prop.create_from_fixture!( name, path ) end def load_records_worker( reader ) reader.each_line do |line| logger.debug " line: >#{line}<" ### fix: use new find_leading_pos! # pos = find_game_pos!( line ) # alias -> rename to find_pos! or better use find_leading_pos!( line ) # match_person!( line ) # match_teams!( line ) record_attribs = { # pos: pos, # team_key: team_key # fix: use team_id } pp record_attribs end # lines.each end # method load_record_worker def load_rosters( name ) path = "#{include_path}/#{name}.txt" logger.info "parsing data '#{name}' (#{path})..." ### SportDB.lang.lang = LangChecker.new.analyze( name, include_path ) reader = LineReader.new( path ) ## for now: use all tracks (later filter/scope by event) # @known_tracks = Track.known_tracks_table ## fix: add @known_teams - for now; use teams (not scoped by event) load_rosters_worker( reader ) Prop.create_from_fixture!( name, path ) end def load_rosters_worker( reader ) reader.each_line do |line| logger.debug " line: >#{line}<" ### fix: use new find_leading_pos! pos = find_game_pos!( line ) # alias -> rename to find_pos! or better use find_leading_pos!( line ) # match_person!( line ) # match_teams!( line ) roster_attribs = { pos: pos, # team_key: team_key # fix: use team_id } pp roster_attribs end # lines.each end # method load_rosters_worker def load_races( name ) path = "#{include_path}/#{name}.txt" logger.info "parsing data '#{name}' (#{path})..." ### SportDB.lang.lang = LangChecker.new.analyze( name, include_path ) reader = LineReader.new( path ) ## for now: use all tracks (later filter/scope by event) @known_tracks = Track.known_tracks_table load_races_worker( reader ) Prop.create_from_fixture!( name, path ) end def load_races_worker( reader ) reader.each_line do |line| logger.debug " line: >#{line}<" ### fix: use new find_leading_pos! pos = find_game_pos!( line ) # alias -> rename to find_pos! or better use find_leading_pos!( line ) match_track!( line ) track_key = find_track!( line ) date = find_date!( line ) race_attribs = { pos: pos, track_key: track_key, # fix: use track_id start_at: date } pp race_attribs end # lines.each end # method load_races_worker def load_teams( name, more_values={} ) path = "#{include_path}/#{name}.txt" logger.info "parsing data '#{name}' (#{path})..." reader = ValuesReader.new( path, more_values ) reader.each_line do |new_attributes, values| Team.create_or_update_from_values( new_attributes, values ) end # each lines Prop.create_from_fixture!( name, path ) end # load_teams private include SportDB::FixtureHelpers def load_fixtures_worker( event_key, reader ) ## assume active activerecord connection ## ## reset cached values @patch_rounds = {} @knockout_flag = false @round = nil @event = Event.find_by_key!( event_key ) logger.debug "Event #{@event.key} >#{@event.title}<" @known_teams = @event.known_teams_table parse_fixtures( reader ) end # method load_fixtures def parse_group( line ) logger.debug "parsing group line: >#{line}<" match_teams!( line ) team_keys = find_teams!( line ) title, pos = find_group_title_and_pos!( line ) logger.debug " line: >#{line}<" group_attribs = { title: title } @group = Group.find_by_event_id_and_pos( @event.id, pos ) if @group.present? logger.debug "update group #{@group.id}:" else logger.debug "create group:" @group = Group.new group_attribs = group_attribs.merge( { event_id: @event.id, pos: pos }) end logger.debug group_attribs.to_json @group.update_attributes!( group_attribs ) @group.teams.clear # remove old teams ## add new teams team_keys.each do |team_key| team = Team.find_by_key!( team_key ) logger.debug " adding team #{team.title} (#{team.code})" @group.teams << team end end def parse_round( line ) logger.debug "parsing round line: >#{line}<" ### todo/fix/check: move cut off optional comment in reader for all lines? why? why not? cut_off_end_of_line_comment!( line ) # cut off optional comment starting w/ # # NB: cut off optional title2 starting w/ // first title2 = find_round_title2!( line ) group_title, group_pos = find_group_title_and_pos!( line ) pos = find_round_pos!( line ) title = find_round_title!( line ) ## NB: use extracted round title for knockout check @knockout_flag = is_knockout_round?( title ) if group_pos.present? @group = Group.find_by_event_id_and_pos!( @event.id, group_pos ) else @group = nil # reset group to no group end logger.debug " line: >#{line}<" ## NB: dummy/placeholder start_at, end_at date ## replace/patch after adding all games for round round_attribs = { title: title, title2: title2, knockout: @knockout_flag } @round = Round.find_by_event_id_and_pos( @event.id, pos ) if @round.present? logger.debug "update round #{@round.id}:" else logger.debug "create round:" @round = Round.new round_attribs = round_attribs.merge( { event_id: @event.id, pos: pos, start_at: Time.utc('1912-12-12'), end_at: Time.utc('1912-12-12') }) end logger.debug round_attribs.to_json @round.update_attributes!( round_attribs ) ### store list of round is for patching start_at/end_at at the end @patch_rounds[ @round.id ] = @round.id end def parse_game( line ) logger.debug "parsing game (fixture) line: >#{line}<" pos = find_game_pos!( line ) match_teams!( line ) team1_key = find_team1!( line ) team2_key = find_team2!( line ) if is_postponed?( line ) postponed = true date_v2 = find_date!( line ) date = find_date!( line ) else postponed = false date_v2 = nil date = find_date!( line ) end scores = find_scores!( line ) logger.debug " line: >#{line}<" ### todo: cache team lookups in hash? team1 = Team.find_by_key!( team1_key ) team2 = Team.find_by_key!( team2_key ) ### check if games exists ## with this teams in this round if yes only update game = Game.find_by_round_id_and_team1_id_and_team2_id( @round.id, team1.id, team2.id ) game_attribs = { score1: scores[0], score2: scores[1], score1ot: scores[2], score2ot: scores[3], score1p: scores[4], score2p: scores[5], play_at: date, play_at_v2: date_v2, postponed: postponed, knockout: @knockout_flag, group_id: @group.present? ? @group.id : nil } game_attribs[ :pos ] = pos if pos.present? if game.present? logger.debug "update game #{game.id}:" else logger.debug "create game:" game = Game.new more_game_attribs = { round_id: @round.id, team1_id: team1.id, team2_id: team2.id } ## NB: use round.games.count for pos ## lets us add games out of order if later needed more_game_attribs[ :pos ] = @round.games.count+1 if pos.nil? game_attribs = game_attribs.merge( more_game_attribs ) end logger.debug game_attribs.to_json game.update_attributes!( game_attribs ) end def parse_fixtures( reader ) reader.each_line do |line| if is_round?( line ) parse_round( line ) elsif is_group?( line ) ## NB: group goes after round (round may contain group marker too) parse_group( line ) else parse_game( line ) end end # lines.each @patch_rounds.each do |k,v| logger.debug "patch start_at/end_at date for round #{k}:" round = Round.find( k ) games = round.games.order( 'play_at asc' ).all ## skip rounds w/ no games ## todo/fix: what's the best way for checking assoc w/ 0 recs? next if games.size == 0 round_attribs = {} ## todo: check for no records ## e.g. if game[0].present? or just if game[0] ?? round_attribs[:start_at] = games[0].play_at round_attribs[:end_at ] = games[-1].play_at logger.debug round_attribs.to_json round.update_attributes!( round_attribs ) end end # method parse_fixtures end # class Reader end # module SportDb