Rails Ruby on Rails → 2.x

Hi,

This is my first bug report so I could be missing something, so please forgive me.

I experience a memory leak when I run the following Rake task. It continues to run for a long time and consumes the memory very fast and as a result slows down.

I suspect that the sentences association (has_and_belongs_to_many) in word.rb is responsible because when I replace sentences_count = word.sentences.size with sentences_count = 0 memory leak is gone.

I found out _why's advice and think this maybe related with it, http://whytheluckystiff.net/arti...

Additionally, I used the following simple memory profiler http://scottstuff.net/blog/artic...

I will attach the output of this profiler for the two different configurations, one with associations and the other with no associations.

desc 'how many times a token set is used.'
task :count_each_word_occurence_roughly => :environment do

  word_count = 0

  words = Word.find(:all, :order => "id")
  words.each do |word|
    parse_tree = word.parse_tree

    parse_tree.root_node.each do |node|
      if node.children.length > 1
        tokens = []
        node.children.each do |child|
          token = Token.find_or_create_by_text(child.name)
          tokens << token.id
        end
        tokens.sort!
        token_ids = tokens.join("-")
        token_set = TokenSet.find_or_initialize_by_token_ids(token_ids)

        sentences_count = word.sentences.size
        if token_set.roughtotal
          token_set.roughtotal += sentences_count
        else
          token_set.roughtotal = sentences_count
        end

        token_set.save
      end
    end

    word_count += 1
    if word_count % 10000 == 0
      print "processed " + word_count.to_s + " words\n"
      GC.start
    end
    # break unless word_count < 2000
  end
end

class Word < ActiveRecord::Base

  #  has_many :word_parses
  #  has_many :parses, :through => :word_parses
  has_and_belongs_to_many :parses, :join_table => "word_parses"

  has_and_belongs_to_many :features, :join_table => "words_features", :uniq => true

  has_and_belongs_to_many :sentences, :join_table => "words_sentences", :uniq => true

  has_many :spellings

  ...

class Sentence < ActiveRecord::Base

  has_and_belongs_to_many :words, :join_table => "words_sentences", :uniq => true
end

schema.rb

ActiveRecord::Schema.define(:version => 5) do

  create_table "features", :force => true do |t|
    t.text "description"
  end

  create_table "parses", :force => true do |t|
    t.text "parse_text", :null => false
  end

  add_index "parses", ["parse_text"], :name => "parse_text"

  create_table "sentences", :force => true do |t|
    t.text "spelling_sequence",      :null => false
    t.text "correct_parse_sequence", :null => false
  end

  create_table "spellings", :force => true do |t|
    t.integer "word_id", :null => false
    t.text    "text",    :null => false
  end

  add_index "spellings", ["word_id"], :name => "word_id"
  add_index "spellings", ["text"], :name => "text"

  create_table "token_sets", :force => true do |t|
    t.integer  "count"
    t.datetime "created_at"
    t.datetime "updated_at"
    t.text     "token_ids"
  end

  add_index "token_sets", ["token_ids"], :name => "token_ids"

  create_table "tokens", :force => true do |t|
    t.text     "text"
    t.datetime "created_at"
    t.datetime "updated_at"
  end

  add_index "tokens", ["text"], :name => "text"

  create_table "word_parses", :force => true do |t|
    t.integer "word_id",  :null => false
    t.integer "parse_id", :null => false
  end

  add_index "word_parses", ["word_id"], :name => "word_id"
  add_index "word_parses", ["parse_id"], :name => "parse_id"

  create_table "words", :force => true do |t|
    t.string "name", :limit => 100, :null => false
  end

  add_index "words", ["name"], :name => "name"

  create_table "words_features", :id => false, :force => true do |t|
    t.integer "word_id",    :null => false
    t.integer "feature_id", :null => false
  end

  add_index "words_features", ["word_id"], :name => "word_id"
  add_index "words_features", ["feature_id"], :name => "feature_id"

  create_table "words_sentences", :id => false, :force => true do |t|
    t.integer "word_id",     :null => false
    t.integer "sentence_id", :null => false
  end

  add_index "words_sentences", ["word_id"], :name => "word_id"
  add_index "words_sentences", ["sentence_id"], :name => "sentence_id"

end