diff --git a/docs/topics/streamfield.rst b/docs/topics/streamfield.rst index 3a3abfa93..41d87c5bc 100644 --- a/docs/topics/streamfield.rst +++ b/docs/topics/streamfield.rst @@ -442,8 +442,59 @@ For block types that simply wrap an existing Django form field, Wagtail provides Migrations ---------- +StreamField definitions within migrations +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + As with any model field in Django, any changes to a model definition that affect a StreamField will result in a migration file that contains a 'frozen' copy of that field definition. Since a StreamField definition is more complex than a typical model field, there is an increased likelihood of definitions from your project being imported into the migration - which would cause problems later on if those definitions are moved or deleted. To mitigate this, StructBlock, StreamBlock and ChoiceBlock implement additional logic to ensure that any subclasses of these blocks are deconstructed to plain instances of StructBlock, StreamBlock and ChoiceBlock - in this way, the migrations avoid having any references to your custom class definitions. This is possible because these block types provide a standard pattern for inheritance, and know how to reconstruct the block definition for any subclass that follows that pattern. If you subclass any other block class, such as ``FieldBlock``, you will need to either keep that class definition in place for the lifetime of your project, or implement a `custom deconstruct method `__ that expresses your block entirely in terms of classes that are guaranteed to remain in place. Similarly, if you customise a StructBlock, StreamBlock or ChoiceBlock subclass to the point where it can no longer be expressed as an instance of the basic block type - for example, if you add extra arguments to the constructor - you will need to provide your own ``deconstruct`` method. + +Migrating RichTextFields to StreamField +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you change an existing RichTextField to a StreamField, and create and run migrations as normal, the migration will complete with no errors, since both fields use a text column within the database. However, StreamField uses a JSON representation for its data, and so the existing text needs to be converted with a data migration in order to become accessible again. For this to work, the StreamField needs to include a RichTextBlock as one of the available block types. The field can then be converted by creating a new migration (``./manage.py makemigrations --empty myapp``) and editing it as follows (in this example, the 'body' field of the ``demo.BlogPage`` model is being converted to a StreamField with a RichTextBlock named ``rich_text``): + +.. code-block:: python + + # -*- coding: utf-8 -*- + from __future__ import unicode_literals + + from django.db import models, migrations + from wagtail.wagtailcore.rich_text import RichText + + + def convert_to_streamfield(apps, schema_editor): + BlogPage = apps.get_model("demo", "BlogPage") + for page in BlogPage.objects.all(): + if page.body.raw_text and not page.body: + page.body = [('rich_text', RichText(page.body.raw_text))] + page.save() + + + def convert_to_richtext(apps, schema_editor): + BlogPage = apps.get_model("demo", "BlogPage") + for page in BlogPage.objects.all(): + if page.body.raw_text is None: + raw_text = ''.join([ + child.value.source for child in page.body + if child.block_type == 'rich_text' + ]) + page.body = raw_text + page.save() + + + class Migration(migrations.Migration): + + dependencies = [ + # leave the dependency line from the generated migration intact! + ('demo', '0001_initial'), + ] + + operations = [ + migrations.RunPython( + convert_to_streamfield, + convert_to_richtext, + ), + ] diff --git a/wagtail/tests/testapp/migrations/0004_streammodel_richtext.py b/wagtail/tests/testapp/migrations/0004_streammodel_richtext.py new file mode 100644 index 000000000..6daa2a302 --- /dev/null +++ b/wagtail/tests/testapp/migrations/0004_streammodel_richtext.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations +import wagtail.wagtailimages.blocks +import wagtail.wagtailcore.blocks +import wagtail.wagtailcore.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('tests', '0003_streammodel'), + ] + + operations = [ + migrations.AlterField( + model_name='streammodel', + name='body', + field=wagtail.wagtailcore.fields.StreamField((('text', wagtail.wagtailcore.blocks.CharBlock()), ('rich_text', wagtail.wagtailcore.blocks.RichTextBlock()), ('image', wagtail.wagtailimages.blocks.ImageChooserBlock()))), + preserve_default=True, + ), + ] diff --git a/wagtail/tests/testapp/models.py b/wagtail/tests/testapp/models.py index e758feba0..298795d79 100644 --- a/wagtail/tests/testapp/models.py +++ b/wagtail/tests/testapp/models.py @@ -11,7 +11,7 @@ from modelcluster.tags import ClusterTaggableManager from wagtail.wagtailcore.models import Page, Orderable from wagtail.wagtailcore.fields import RichTextField, StreamField -from wagtail.wagtailcore.blocks import CharBlock +from wagtail.wagtailcore.blocks import CharBlock, RichTextBlock from wagtail.wagtailadmin.edit_handlers import FieldPanel, MultiFieldPanel, InlinePanel, PageChooserPanel, TabbedInterface, ObjectList from wagtail.wagtailimages.edit_handlers import ImageChooserPanel from wagtail.wagtaildocs.edit_handlers import DocumentChooserPanel @@ -407,5 +407,6 @@ class CustomImageWithAdminFormFields(AbstractImage): class StreamModel(models.Model): body = StreamField([ ('text', CharBlock()), + ('rich_text', RichTextBlock()), ('image', ImageChooserBlock()), ]) diff --git a/wagtail/wagtailcore/blocks/stream_block.py b/wagtail/wagtailcore/blocks/stream_block.py index 9b4150109..db973c46d 100644 --- a/wagtail/wagtailcore/blocks/stream_block.py +++ b/wagtail/wagtailcore/blocks/stream_block.py @@ -260,7 +260,7 @@ class StreamValue(collections.Sequence): """ return self.block.name - def __init__(self, stream_block, stream_data, is_lazy=False): + def __init__(self, stream_block, stream_data, is_lazy=False, raw_text=None): """ Construct a StreamValue linked to the given StreamBlock, with child values given in stream_data. @@ -273,11 +273,18 @@ class StreamValue(collections.Sequence): Passing is_lazy=False means that stream_data consists of immediately usable native values. In this mode, stream_data is a list of (type_name, value) tuples. + + raw_text exists solely as a way of representing StreamField content that is + not valid JSON; this may legitimately occur if an existing text field is + migrated to a StreamField. In this situation we return a blank StreamValue + with the raw text accessible under the `raw_text` attribute, so that migration + code can be rewritten to convert it as desired. """ self.is_lazy = is_lazy self.stream_block = stream_block # the StreamBlock object that handles this value self.stream_data = stream_data # a list of (type_name, value) tuples self._bound_blocks = {} # populated lazily from stream_data as we access items through __getitem__ + self.raw_text = raw_text def __getitem__(self, i): if i not in self._bound_blocks: diff --git a/wagtail/wagtailcore/fields.py b/wagtail/wagtailcore/fields.py index 73fa59912..3b9e3fc09 100644 --- a/wagtail/wagtailcore/fields.py +++ b/wagtail/wagtailcore/fields.py @@ -5,7 +5,7 @@ import json from django.db import models from django import forms from django.core.serializers.json import DjangoJSONEncoder -from django.utils.six import with_metaclass +from django.utils.six import with_metaclass, string_types from wagtail.wagtailcore.rich_text import DbWhitelister, expand_db_html from wagtail.utils.widgets import WidgetWithScript @@ -69,17 +69,16 @@ class StreamField(with_metaclass(models.SubfieldBase, models.Field)): return StreamValue(self.stream_block, []) elif isinstance(value, StreamValue): return value - else: # assume string + elif isinstance(value, string_types): try: unpacked_value = json.loads(value) except ValueError: # value is not valid JSON; most likely, this field was previously a # rich text field before being migrated to StreamField, and the data - # was left intact in the migration. Return an empty stream instead. - - # TODO: keep this raw text data around as a property of the StreamValue - # so that it can be retrieved in data migrations - return StreamValue(self.stream_block, []) + # was left intact in the migration. Return an empty stream instead + # (but keep the raw text available as an attribute, so that it can be + # used to migrate that data to StreamField) + return StreamValue(self.stream_block, [], raw_text=value) if unpacked_value is None: # we get here if value is the literal string 'null'. This should probably @@ -88,9 +87,27 @@ class StreamField(with_metaclass(models.SubfieldBase, models.Field)): return StreamValue(self.stream_block, []) return self.stream_block.to_python(unpacked_value) + else: + # See if it looks like the standard non-smart representation of a + # StreamField value: a list of (block_name, value) tuples + try: + [None for (x, y) in value] + except (TypeError, ValueError): + # Give up trying to make sense of the value + raise TypeError("Cannot handle %r (type %r) as a value of StreamField" % (value, type(value))) + + # Test succeeded, so return as a StreamValue-ified version of that value + return StreamValue(self.stream_block, value) def get_prep_value(self, value): - return json.dumps(self.stream_block.get_prep_value(value), cls=DjangoJSONEncoder) + if isinstance(value, StreamValue) and not(value) and value.raw_text is not None: + # An empty StreamValue with a nonempty raw_text attribute should have that + # raw_text attribute written back to the db. (This is probably only useful + # for reverse migrations that convert StreamField data back into plain text + # fields.) + return value.raw_text + else: + return json.dumps(self.stream_block.get_prep_value(value), cls=DjangoJSONEncoder) def formfield(self, **kwargs): """ diff --git a/wagtail/wagtailcore/tests/test_streamfield.py b/wagtail/wagtailcore/tests/test_streamfield.py index b35c1146f..c85a2df9d 100644 --- a/wagtail/wagtailcore/tests/test_streamfield.py +++ b/wagtail/wagtailcore/tests/test_streamfield.py @@ -9,6 +9,8 @@ from wagtail.wagtailcore import blocks from wagtail.wagtailcore.fields import StreamField from wagtail.wagtailimages.models import Image from wagtail.wagtailimages.tests.utils import get_test_image_file +from wagtail.wagtailcore.blocks import StreamValue +from wagtail.wagtailcore.rich_text import RichText class TestLazyStreamField(TestCase): @@ -21,6 +23,7 @@ class TestLazyStreamField(TestCase): {'type': 'text', 'value': 'foo'}])) self.no_image = StreamModel.objects.create(body=json.dumps([ {'type': 'text', 'value': 'foo'}])) + self.nonjson_body = StreamModel.objects.create(body="

hello world

") def test_lazy_load(self): """ @@ -99,3 +102,29 @@ class TestSystemCheck(TestCase): self.assertEqual(errors[0].id, 'wagtailcore.E001') self.assertEqual(errors[0].hint, "Block names cannot contain spaces") self.assertEqual(errors[0].obj, InvalidStreamModel._meta.get_field('body')) + + +class TestStreamValueAccess(TestCase): + def setUp(self): + self.json_body = StreamModel.objects.create(body=json.dumps([ + {'type': 'text', 'value': 'foo'}])) + self.nonjson_body = StreamModel.objects.create(body="

hello world

") + + def test_can_read_non_json_content(self): + """StreamField columns should handle non-JSON database content gracefully""" + self.assertIsInstance(self.nonjson_body.body, StreamValue) + # the main list-like content of the StreamValue should be blank + self.assertFalse(self.nonjson_body.body) + # the unparsed text content should be available in raw_text + self.assertEqual(self.nonjson_body.body.raw_text, "

hello world

") + + def test_can_assign_as_list(self): + self.json_body.body = [('rich_text', RichText("

hello world

"))] + self.json_body.save() + + # the body should now be a stream consisting of a single rich_text block + fetched_body = StreamModel.objects.get(id=self.json_body.id).body + self.assertIsInstance(fetched_body, StreamValue) + self.assertEqual(len(fetched_body), 1) + self.assertIsInstance(fetched_body[0].value, RichText) + self.assertEqual(fetched_body[0].value.source, "

hello world

")