From: Joseph Spiros Date: Mon, 28 Feb 2011 09:25:59 +0000 (+0000) Subject: Added support for ETag and Last-Modified headers. X-Git-Url: http://git.ithinksw.org/~jspiros/reader.git/commitdiff_plain/9b96433a50883314722dd38216dc38e8915f2a4f?ds=inline;pf=~jspiros Added support for ETag and Last-Modified headers. --- diff --git a/migrations/0002_auto__add_field_feed_etag__add_field_feed_modified__add_field_feed_ali.py b/migrations/0002_auto__add_field_feed_etag__add_field_feed_modified__add_field_feed_ali.py new file mode 100644 index 0000000..e86b3fe --- /dev/null +++ b/migrations/0002_auto__add_field_feed_etag__add_field_feed_modified__add_field_feed_ali.py @@ -0,0 +1,100 @@ +# encoding: utf-8 +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + +class Migration(SchemaMigration): + + def forwards(self, orm): + + # Adding field 'Feed.etag' + db.add_column('reader_feed', 'etag', self.gf('django.db.models.fields.CharField')(max_length=255, null=True, blank=True), keep_default=False) + + # Adding field 'Feed.modified' + db.add_column('reader_feed', 'modified', self.gf('django.db.models.fields.DateTimeField')(null=True, blank=True), keep_default=False) + + # Adding field 'Feed.alive' + db.add_column('reader_feed', 'alive', self.gf('django.db.models.fields.BooleanField')(default=True), keep_default=False) + + + def backwards(self, orm): + + # Deleting field 'Feed.etag' + db.delete_column('reader_feed', 'etag') + + # Deleting field 'Feed.modified' + db.delete_column('reader_feed', 'modified') + + # Deleting field 'Feed.alive' + db.delete_column('reader_feed', 'alive') + + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'reader.entry': { + 'Meta': {'object_name': 'Entry'}, + 'content': ('django.db.models.fields.TextField', [], {}), + 'feed': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'entries'", 'to': "orm['reader.Feed']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'published': ('django.db.models.fields.DateTimeField', [], {}), + 'summary': ('django.db.models.fields.TextField', [], {}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'updated': ('django.db.models.fields.DateTimeField', [], {}), + 'uri': ('django.db.models.fields.CharField', [], {'max_length': '255'}) + }, + 'reader.feed': { + 'Meta': {'object_name': 'Feed'}, + 'alive': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'etag': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'modified': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'updated': ('django.db.models.fields.DateTimeField', [], {}), + 'url': ('django.db.models.fields.URLField', [], {'max_length': '200'}) + }, + 'reader.subscription': { + 'Meta': {'object_name': 'Subscription'}, + 'custom_title': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'feed': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'subscriptions'", 'to': "orm['reader.Feed']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'reader_subscriptions'", 'to': "orm['auth.User']"}) + } + } + + complete_apps = ['reader'] diff --git a/models.py b/models.py index 28adb66..806b3ef 100644 --- a/models.py +++ b/models.py @@ -6,6 +6,9 @@ class Feed(models.Model): url = models.URLField() title = models.CharField(max_length=255) updated = models.DateTimeField() + etag = models.CharField(max_length=255, blank=True, null=True) + modified = models.DateTimeField(blank=True, null=True) + alive = models.BooleanField(default=True) class Entry(models.Model): diff --git a/utils.py b/utils.py index 2215ec8..e58dbdb 100644 --- a/utils.py +++ b/utils.py @@ -4,11 +4,22 @@ import feedparser def refresh_feed(feed, save=True): - parsed = feedparser.parse(feed.url) - feed.title = parsed.feed.get('title', feed.url) - feed.updated = datetime.datetime(*(parsed.feed.get('updated_parsed', datetime.datetime.now().timetuple())[0:6])) - if save: - feed.save() + if feed.alive: + parsed = feedparser.parse(feed.url, etag=feed.etag, modified=(feed.modified.timetuple() if feed.modified else None)) + if parsed.get('status', None) == 304: + return + if parsed.get('status', None) == 301 and parsed.has_key('href'): + feed.url = parsed.href + if parsed.get('status', None) == 410: + feed.alive = False + if parsed.has_key('etag'): + feed.etag = parsed.etag + if parsed.has_key('modified'): + feed.modified = datetime.datetime(*(parsed.modified[0:6])) + feed.title = parsed.feed.get('title', feed.url) + feed.updated = datetime.datetime(*(parsed.feed.get('updated_parsed', datetime.datetime.now().timetuple())[0:6])) + if save: + feed.save() def add_feed(url):