Skip to content

Migration workflow

Jump to bottom
Johnny Mariéthoz edited this page Feb 12, 2026 · 55 revisions

v1.12.3 --> v1.12.4

docker

  • Change grobid version to grobid/grobid:0.8.2-crf

projects data

  • for all projects where organisation is not "hepvs", delete validation field
"""Remove validation field from projects not belonging to hepvs.

Usage (from flask shell):
    from scripts.remove_validation_non_hepvs import run
    run(dry_run=True)   # simulation
    run()               # exécution réelle
"""

from elasticsearch_dsl import Q, Search
from invenio_db import db
from invenio_pidstore.models import PersistentIdentifier
from invenio_search import current_search

from sonar.resources.projects.api import Record
from sonar.proxies import sonar

def run(dry_run=False):
    """Remove the validation field from all projects not belonging to hepvs."""
    client = current_search.client
    service = sonar.service("projects")

    # Search for projects that:
    # - do NOT belong to organisation "hepvs"
    # - have a validation field
    search = Search(using=client, index="projects").query(
        Q(
            "bool",
            must=[
                Q("exists", field="metadata.validation"),
            ],
            must_not=[
                Q("term", metadata__organisation__pid="hepvs"),
            ],
        )
    )

    total = search.count()
    print(f"Found {total} project(s) to update.")

    if total == 0:
        print("Nothing to do.")
        return

    updated = 0
    errors = 0

    for hit in search.scan():
        project_id = hit.meta.id
        try:
            record = Record.get_record(project_id)
        except Exception as exc:
            print(f"  ERROR loading project {project_id}: {exc}")
            errors += 1
            continue

        org_info = hit.to_dict().get("metadata", {}).get("organisation", {})
        org_pid = org_info.get("pid", "unknown")

        if "validation" not in record.get("metadata", {}):
            print(f"  Skipping {project_id} (org={org_pid}): no validation field in DB.")
            continue

        print(f"  Removing validation from project {project_id} (org={org_pid})")

        if not dry_run:
            del record["metadata"]["validation"]
            record.commit()
            db.session.commit()
            service.indexer.index(record)

        updated += 1

    if dry_run:
        print(f"\n[DRY RUN] Would have updated {updated} project(s). {errors} error(s).")
    else:
        print(f"\nDone. Updated {updated} project(s). {errors} error(s).")

v1.11.0 --> v1.12.0

Alembic

uv run invenio alembic stamp 428b919be0ea
uv run invenio alembic upgrade

Update mapping

from invenio_search import current_search_client
current_search_client.indices.create('records-record-v1.0.0')
uv run invenio rero es index update-mapping

Documents

Remove _oai in documents.

# estimation for 300'000 records: 5h
from invenio_pidstore.models import PersistentIdentifier
from rero_invenio_base.modules.tasks import run_on_worker
from rero_invenio_base.modules.utils import chunk


code = '''
def pop_oai(_ids):
    from sonar.modules.documents.api import DocumentRecord
    from invenio_db import db
    success = 0
    failed = 0
    for uuid in _ids:
        try:
            record = DocumentRecord.get_record(uuid)
            if record.get('_oai'):
                record['_oai'].pop('updated', None)
                record['_oai'].pop('sets', None)
                record.commit()
                db.session.commit()
                record.reindex()
                success += 1
        except Exception as err:
            print(f"Exception {uuid}: {err}")
            failed += 1
    return {"success": success, "failed": failed}
'''


parallel = 7
ids = [pid.object_uuid for pid in PersistentIdentifier.query.filter_by(pid_type='doc').filter_by(status='R').all()]
for count, c in enumerate(chunk([str(val) for val in ids], len(ids) // parallel), 1):
    res = run_on_worker.delay(code, 'pop_oai', c)
    print('documents', count, len(c), res)
index_name=`uv run invenio rero es index info -i documents |grep -v percolator`
echo $index_name
uv run invenio rero es index move documents $index_name documents-document-v1.0.0-20250923 -v
uv run invenio index delete $index_name --yes-i-know

Deposits

Remove category from deposit.

from sonar.modules.deposits.api import DepositRecord
n = 0
for pid in PersistentIdentifier.query.filter_by(
            pid_type='depo').filter_by(status='R').all():
    try:
        record = DepositRecord.get_record(pid.object_uuid) 
        n += 1
        changed = False
        for f in record.get('_files', []):
            if 'category' in f:
                del f['category']
                changed = True
            if 'embargo' in f:
                del f['embargo']
                changed = True
        if changed:
            record.commit()
            db.session.commit()
            record.reindex()
    except:
        print(f"Failed to update record {pid.pid_value}")

print(f"Updated {n} records")
index_name=`uv run invenio rero es index info -i deposits | grep -v percolator`
echo $index_name
uv run invenio rero es index move deposits $index_name deposits-deposit-v1.0.0-20250923 -v
uv run invenio index delete $index_name --yes-i-know

Organisations

index_name=`uv run invenio rero es index info -i organisations | grep -v percolator`
echo $index_name
uv run invenio rero es index move organisations $index_name organisations-organisation-v1.0.0-20250923 -v
uv run invenio index delete $index_name --yes-i-know

Collections

index_name=`uv run invenio rero es index info -i collections | grep -v percolator`
echo $index_name
uv run invenio rero es index move collections $index_name collections-collection-v1.0.0-20250923 -v
uv run invenio index delete $index_name --yes-i-know

ConfigMap

  • replace INVENIO_SQLALCHEMY_POOL_RECYCLE by INVENIO_SQLALCHEMY_ENGINE_OPTIONS: '{ "pool_recycle": 360 }'
  • set the config in the sqlalchemy container/server by setting PGOPTIONS to -c statement_timeout=30s -c idle_in_transaction_session_timeout=60s
  • add IDP_CERTIFICATES_DIR: /invenio/storage/prod/data/idp_certificates
  • add SONAR_FILES_PATH: /invenio/storage/files
  • add PYTHONWARNINGS: ignore

Post migration

Clone this wiki locally