From ce5ae2b9a089d41a72929d51bf5deb99efa124a0 Mon Sep 17 00:00:00 2001 From: David Wallace Date: Wed, 9 Sep 2026 21:19:58 +0200 Subject: [PATCH 01/24] Update translations Signed-off-by: David Wallace --- .../locale/de/LC_MESSAGES/django.mo | Bin 1610 -> 1762 bytes .../locale/de/LC_MESSAGES/django.po | 5 +++++ .../locale/es/LC_MESSAGES/django.mo | Bin 992 -> 1463 bytes .../locale/es/LC_MESSAGES/django.po | 5 +++++ .../locale/fr/LC_MESSAGES/django.mo | Bin 1017 -> 1441 bytes .../locale/fr/LC_MESSAGES/django.po | 5 +++++ .../locale/it/LC_MESSAGES/django.mo | Bin 986 -> 1418 bytes .../locale/it/LC_MESSAGES/django.po | 5 +++++ 8 files changed, 20 insertions(+) diff --git a/rdmo_plugins_statistics/locale/de/LC_MESSAGES/django.mo b/rdmo_plugins_statistics/locale/de/LC_MESSAGES/django.mo index 937fecdfad051401589fa21601f25c517723b548..6fe8cbfe794d779d039fc5872cf8b6abbc46d797 100644 GIT binary patch delta 743 zcmYk(ze^lJ6u|Mh)ARRfRQwes48nmLun-#|DHSXf{{va>hP#dHp6-sQSLCWR7B*sM zC6Z1^B3dZe*oYQ378WWfh$$_!(jwn?PLW~XeD2K+Z{F;lHQlGnw{6v%iqYrd4Ku`b zf{C$^RB8klF^xM|i+@~xh>hf@Zv7Ig$*-{r@2~+M-1-xy$X~Ecsj_g-W5uNe^Kf^qcr%2I#E_CLH3!2e%hiV zHI1^-!)EkR8eG9T+&~_+#Y7FYjndFhm;Xi{wa+9C9APh>qrAaKxBh~)oUc+f?+2$* z4%R3g58xz@y8JuJfou2)*HPZguJah>fD4ox-k|%6WJ0XB1US6pTnyxe_zo}mt-OTkw=iXRyDDkCGT=v7f_N<;S##Y;c(V2n3pZD!x z{Js>KtO;$fsmf0L^%`c!pT9l5L*+*I_Ljir?eCp eTe8|j{2K)9&@)nG&Km87c~dfe-VSd(q|86C_GFj< delta 592 zcmXZZy)Q#y6vy$?+iJbEUT#xVMJyz(tpgKV2MddY(OY6_;vb-c&5EJM4kQMN+=0zv zk}ibBAR=M0i5SKAcY99mc|JW)?tPx~JblmIXB)3!>cof=IZpnNQ)D~AK&%I5RouWF z9%BYuUSD7z^^MQpVT!tqNqoRweD?V_?4tg_(5zu!{1{{64+pTwS6M8hHk!pjT=04s zwcx6|iF|5xEaEO|okLV(r>J$#P>o&UB!`P|hW+h{SvrZ^(u;Z8>&FtRu?lu$4S8&d zfp2XE)ySIHTgYQO3~Hc`y45|@%{6`g9JS9ihU{;*KJkd^_{IH1ZS;*3_=~!ULb~HL zYT-H5$!h5T8kN})sRk4}px#mFP+CWaoF8V;B6(6>D|C&c@vqK_)Rcnt;}~J=Aq&xB PP)S5r$%Ei%KfV1A-0Lj* diff --git a/rdmo_plugins_statistics/locale/de/LC_MESSAGES/django.po b/rdmo_plugins_statistics/locale/de/LC_MESSAGES/django.po index 2cb8482..4e24d2e 100644 --- a/rdmo_plugins_statistics/locale/de/LC_MESSAGES/django.po +++ b/rdmo_plugins_statistics/locale/de/LC_MESSAGES/django.po @@ -130,5 +130,10 @@ msgstr "Gesamt im angezeigten Zeitraum" msgid "Total in system" msgstr "Gesamt im System" +#: templates/rdmo_plugins_statistics/statistics.html +#, python-format +msgid "Showing at most the %(limit)s most recently created projects." +msgstr "Es werden höchstens die %(limit)s zuletzt erstellten Projekte angezeigt." + #~ msgid "Number of users over time" #~ msgstr "Summierte Nutzer" diff --git a/rdmo_plugins_statistics/locale/es/LC_MESSAGES/django.mo b/rdmo_plugins_statistics/locale/es/LC_MESSAGES/django.mo index bf2c8c2e800db663cbd637f88d16c3a4cd3923b3..30056ba9a7ba898103dd01c54c803d6c8ee7bd44 100644 GIT binary patch literal 1463 zcmY+DPmdcl7{=Yw@+U3OmLk-OikCyHQZ&nMd#SPwbec@sh)l91*_F!8JLAn7&5VQX zX*cR;;7ISiR@y^*;DCgL6fPY20vr%<YD^T7= zeINBF)Jv$@F*GQ@gKvU=fUkjnFZ(aRQ|O;OUdPXZub_VcY=D=*6W~?wBp86_z#HI2 zunY2f66EpTti!S_HumlVV!EE+zy1MnU2 zF~~Zff>=U42O+cC15g!6t~9<5at0BifE>!rBr(5MB6Gb6&*Pj(ZS4EPwnxJ=vtGD z*c5(W49a|0Su%Sxv*s(6c=w->Ra5P}vMR;e$~y5yDJ}Y9bTlBw`{to8rX)S(#(DDl ziY{NxbgunJt`1q1sKRG+N@_Vrsn*2UOOF&fabjZdU_+z}O0}D1a*n(+Wwl9HBW?~s zx!BR_A??n^ee6|i)yjij8tg!&Hbs-R!uIFk-H3Lg(YQApG)9VdAH;5|)8I}yb-~0m zsj)Q-M(Tm)Z-Nevs!8{wa1_MDpcB%)VKeT1{N->EHTu%|U}9zAGU<(NQg^%Gr$HD; zbSsL(UjKI3>5QWB_)|GtuNgaC&4Gfr7e_1E8*A&0 zHZBI|6-?$c=<({ne}co%#U>^D(mLf^+mmi^b2X1;?I{~XMPgFiMU!ssYTww(N-MLV zYi#bCRLrWbYu(tS`n6S1t3@018Lbn>>BAOYu3W*}W!hM;3Qt(agbhr%zS)5P3R7K< zDfH5Tizl$45!(&}iW2GTfP+VzXPYLl*PhP2ttRX#Y@#-1YD8yoD3`{BXW5z^R$fY|~K`LXzE)GqKB={Su^tG$& TQOEyNE(Lr_Fr)|<&y4sNduMKa delta 545 zcmYk&yGjE=6oBE$x-pnUj8Q_=hzll?v=)MBVPRuujMg#4RaVeQcD-N|uunAzLTr2k zLB!I^QUp_aAt07kK0y4Bj^bfvzBy-i=dw4kPNeagv`-b0rf2BSl9-sr^N>>g_<#|7 zG5I^j$UjZJi#B=0QmO~z7{vrieGJ3sU=pWrRH=rVWfG^bin2ik6X==z7-gXoV-soW zj9~;@IDpqE-`}BZc#jf~=6M_Y$lq`fKQP1os>`I8LPm~FOrk81MLAdj<>WI+Q}YbN zxPTM5CPiZ3cwjt4+3*OzFM#qVn<(qGaJ5Xvd3(T}d;s5{u diff --git a/rdmo_plugins_statistics/locale/es/LC_MESSAGES/django.po b/rdmo_plugins_statistics/locale/es/LC_MESSAGES/django.po index b915b03..43797b1 100644 --- a/rdmo_plugins_statistics/locale/es/LC_MESSAGES/django.po +++ b/rdmo_plugins_statistics/locale/es/LC_MESSAGES/django.po @@ -132,6 +132,11 @@ msgstr "Total en el período mostrado" msgid "Total in system" msgstr "Total en el sistema" +#: templates/rdmo_plugins_statistics/statistics.html +#, python-format +msgid "Showing at most the %(limit)s most recently created projects." +msgstr "Se muestran como máximo los %(limit)s proyectos creados más recientemente." + #, fuzzy #~| msgid "Number of projects" #~ msgid "Number of users over time" diff --git a/rdmo_plugins_statistics/locale/fr/LC_MESSAGES/django.mo b/rdmo_plugins_statistics/locale/fr/LC_MESSAGES/django.mo index 86b4a3044229f1b27a7a51381a816bcde9355a50..93f302ef67e152de998d5ca767a0e875b6ff8129 100644 GIT binary patch literal 1441 zcmYk4%Z?jG6ow1Pa52|}ixfn}0SPey^*~?~Jeeqs-A+t=8Enspz^d#n_Y~@`GF8>d zSZsI!#0nPh0(cvV4H6QPS@Hxd5Nz2*ViU3BKXzw4rLO+^RG+GIssGP&XTB8}FJiue z`5Wd1%={s|F#ZCc2LA>h1MgM#eeg-xTMw_-&w`J@J`dKw3*Z@W8{7gT@ErI)_%hf6 z`8)}-zXtcfP}z4uJmMF;&V#>#&w_t|yzd{7&;J*Uk;{GX8Q33S@d@w-$Z-bXYhVFB z3Vs9P5#QnU68Hm%t>SLA{u6i>_Rrw!;O`)xcMs%x{{s_n3q?Hz-UNC7C*Z5#XCUwY zvikou$Z@|1k>;ufU^RFj`#z4zF?jHq?9aVKDDgZd$L0Is@d75_2M_M&gTX!I@f;>X zt$M+I5^qsyQInI@6$RKl@Js_1{W?0RYbd<`Vyl6NDrlKk5A~~K}7pO5j6fI{9 z(X}QViC$TZl_NW)nX{j(N${tZoXWHh%Bd7*E9b?>rF5{x;N*uG9@#tEWF!L>)&~kl zir(1Hb)myWzg}``qD;u=G+C!}5^6^bgA53vCti#!9u!2{P^$ebmvh9ODW`3^`QztE zs1S!*-J$VZ+(NBlf1P>MPNPF)sjaEge%!bjUr*>T84TO~UTvVb`l#zOokky)nU6-c zPPP4hG*GuS?}?gdRh@1n@gVBL?b7S&tOE^G5xuU7ggH!N3QQva0T!@cs=_>WXPnI-AMvx0Dv8kX#4b9nNyxzK!A*--j zd9$oiP^I&v%vzIfxqxDvE|f<|k!+SlJe_JZD&s`_DJfP5Lxh}_N5Y;P;R!bD({pNt zJk~{7Y=rp*m%99_5}+?psHE9)!KzdwSFx50RsBy(Vu3DyEL4>TDxN{2u`%rRKS{}O A6aWAK delta 545 zcmYk&ze~eF6u|LoObxbGYlm8YfKgQ1O&vsWbrBIN{umrArNmM}D@`Vsf}`5$CQln5DVhYbgN_FBL zI{0Yv7DmZGO#KUOa>r7t4PzL=I7)p0!{}lH$FX0js+wgIqp*t7Adhh@nS38*qXT0N zxzs7QK0L#2yheHd2BqOGO59@?Hn9U=kjCm2hv}~zj@wQlBgZBtP&SyvBu*o1)sm@a zv4?yON3e(zhsGo0F-pT4%AYzx`Or(0{jP9mnMs3*Y}hmlS||s4$B6qcR)Ty`@`F;r zcelzFcP#Cf0x!3l-|;g3-$MNynzI5=r>uq1Y^7YRc)stZ$1;j!+baaWx_dtlb`h&_H#nMpvwQVoc>Npl5l0sQ diff --git a/rdmo_plugins_statistics/locale/fr/LC_MESSAGES/django.po b/rdmo_plugins_statistics/locale/fr/LC_MESSAGES/django.po index a20f567..cb1a786 100644 --- a/rdmo_plugins_statistics/locale/fr/LC_MESSAGES/django.po +++ b/rdmo_plugins_statistics/locale/fr/LC_MESSAGES/django.po @@ -131,6 +131,11 @@ msgstr "Total sur la période affichée" msgid "Total in system" msgstr "Total du système" +#: templates/rdmo_plugins_statistics/statistics.html +#, python-format +msgid "Showing at most the %(limit)s most recently created projects." +msgstr "Affichage des %(limit)s projets créés les plus récents au maximum." + #, fuzzy #~| msgid "Number of projects" #~ msgid "Number of users over time" diff --git a/rdmo_plugins_statistics/locale/it/LC_MESSAGES/django.mo b/rdmo_plugins_statistics/locale/it/LC_MESSAGES/django.mo index 766b9d0eff3bb4370e83458ce4e076d0a3ca846a..71cfb2cd368454a354b2345c038e43248aa15302 100644 GIT binary patch literal 1418 zcmY+DOOG2x6om^&c$jCxLkbXaSwxzEdLSUAs3%70*zLi>kHIe~?8@$%cA@SntEzeu zXNkmu9UFE?Z1@Fi5DO$E_y_EeK(Iq>kl19)xzlz$l)CzJz51Se>-hJJ=e`tZFQUJS z{xkYz^x`4B(Eb3Q2LA*f1OM5K?}JZbeE#8j{sQ<2#+Sebco{qg?t$mQ2)qcs3%(3? zK>nTtdA|mmVAzbm1@VX<@VW&42tEt`3i7$XLH_=~;4Nsm4_?4H#^mE*4}1xnf}C3c zJ_>%i;pZUh{{qBO@fC7i@-oM4p`@cs2IWP|DKHalxfbc&m34tyg^LpfOUMebo24&Z1jY?xTr~sc|%n zM(RZKnW&9Y#dJGOMp1tlwG;Ya827txemoqcjh^%&nmB2EA%k@>b*8-@4U&FJH`0F6 z?cGe;?NK@&zmKgZuW%93M9DIyUNShGvXUFU-pxYqTbJ+lKtaFTPj|Af?Z4e<;bKs) zXtG{GPf&NmTPTJ$2eep9=T&GrJ|L^`HS`8C#d#8a>xLb6GJBy3qITS;~~AMQypIh6QxF!i7sZ z)0T!#y=1@(Lj7opVsA=Go8 z*S>^!=q)~%;5&F0euTRJ6V&^@KyBaPIrtr(fCumb`~%O!Sp<6=HZ*9oYjyuR)cn>U zmeMV#=dMGI|2AyE9jGm<&1)SziSYtz{6nbcy?`hcy@FTaYpCaafM?)msQ3JU^Jc2; zMjItW=>*et-Yl;KiTpUX8Lx)Jg|ueM-0oXRZ98RZ+GtbP8*9p7dl*gGdXw15ftsFS zLYw&FGoYp?YB@ZJ-bii|2}NoLIKN;oE}6`P0WQvHPxz^6 z59kpR!!I+r8*OsAv}IBUM-QWqRhQ9x(86aH1sKok7TU-W?|)kP;L-p9 diff --git a/rdmo_plugins_statistics/locale/it/LC_MESSAGES/django.po b/rdmo_plugins_statistics/locale/it/LC_MESSAGES/django.po index 1af9f33..64f5597 100644 --- a/rdmo_plugins_statistics/locale/it/LC_MESSAGES/django.po +++ b/rdmo_plugins_statistics/locale/it/LC_MESSAGES/django.po @@ -131,6 +131,11 @@ msgstr "Totale nel periodo visualizzato" msgid "Total in system" msgstr "Totale nel sistema" +#: templates/rdmo_plugins_statistics/statistics.html +#, python-format +msgid "Showing at most the %(limit)s most recently created projects." +msgstr "Vengono visualizzati al massimo i %(limit)s progetti creati più di recente." + #, fuzzy #~| msgid "Number of projects" #~ msgid "Number of users over time" From 6edba3cb34576452f5d7eb02bb12f45292d7b770 Mon Sep 17 00:00:00 2001 From: David Wallace Date: Wed, 9 Sep 2026 21:21:32 +0200 Subject: [PATCH 02/24] build: update py versions and add test deps Signed-off-by: David Wallace --- pyproject.toml | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 50f40ea..e249b5d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,34 +17,44 @@ license = {text = "Apache-2.0"} authors = [ {name = "RDMO Arbeitsgemeinschaft", email = "rdmo-team@listserv.dfn.de"}, ] -requires-python = ">=3.8" +requires-python = ">=3.10" classifiers = [ "Development Status :: 5 - Production/Stable", "Environment :: Web Environment", - "Framework :: Django :: 4.2", + "Framework :: Django :: 5.2", "Intended Audience :: Science/Research", "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", ] dynamic = [ "version", ] dependencies = [ - "rdmo" + "rdmo>=2.5,<4" ] [project.optional-dependencies] +fixtures = ["model-bakery>=1.20,<2"] dev = [ + "model-bakery>=1.20,<2", "pre-commit", + "pytest>=8,<10", + "pytest-django>=4.8,<5", ] +[tool.pytest.ini_options] +DJANGO_SETTINGS_MODULE = "tests.settings" +testpaths = ["tests"] +pythonpath = ["."] +addopts = "-ra" + [project.urls] documentation = "https://rdmo.readthedocs.io/en/latest/plugins/" homepage = "https://rdmorganiser.github.io" @@ -62,7 +72,7 @@ include = ["rdmo_plugins_statistics*"] version = {attr = "rdmo_plugins_statistics.__version__"} [tool.ruff] -target-version = "py38" +target-version = "py310" line-length = 120 select = [ "B", # flake8-bugbear From 0437526c96da7f4524cb0fc0eab20d08bd2765ab Mon Sep 17 00:00:00 2001 From: David Wallace Date: Wed, 9 Sep 2026 21:23:52 +0200 Subject: [PATCH 03/24] tests: add pytests and model bakery fixture creation commands Signed-off-by: David Wallace --- .../management/commands/__init__.py | 0 .../create_statistics_test_projects.py | 52 +++ .../commands/create_statistics_test_users.py | 50 +++ rdmo_plugins_statistics/testing.py | 335 ++++++++++++++++++ tests/__init__.py | 1 + tests/settings.py | 21 ++ tests/test_generator.py | 147 ++++++++ tests/test_statistics.py | 125 +++++++ tests/test_user_generator.py | 87 +++++ tests/urls.py | 11 + 10 files changed, 829 insertions(+) create mode 100644 rdmo_plugins_statistics/management/commands/__init__.py create mode 100644 rdmo_plugins_statistics/management/commands/create_statistics_test_projects.py create mode 100644 rdmo_plugins_statistics/management/commands/create_statistics_test_users.py create mode 100644 rdmo_plugins_statistics/testing.py create mode 100644 tests/__init__.py create mode 100644 tests/settings.py create mode 100644 tests/test_generator.py create mode 100644 tests/test_statistics.py create mode 100644 tests/test_user_generator.py create mode 100644 tests/urls.py diff --git a/rdmo_plugins_statistics/management/commands/__init__.py b/rdmo_plugins_statistics/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/rdmo_plugins_statistics/management/commands/create_statistics_test_projects.py b/rdmo_plugins_statistics/management/commands/create_statistics_test_projects.py new file mode 100644 index 0000000..2e0a8c8 --- /dev/null +++ b/rdmo_plugins_statistics/management/commands/create_statistics_test_projects.py @@ -0,0 +1,52 @@ +import json +from datetime import datetime + +from django.contrib.auth import get_user_model +from django.contrib.sites.models import Site +from django.core.exceptions import ObjectDoesNotExist +from django.core.management.base import BaseCommand, CommandError + +from rdmo.questions.models import Catalog + +from rdmo_plugins_statistics.testing import create_test_projects + + +class Command(BaseCommand): + help = 'Create catalog-based test projects, memberships and interview values.' + + def add_arguments(self, parser): + parser.add_argument('--catalog-id', type=int, required=True) + parser.add_argument('--site-id', type=int) + parser.add_argument('--owner', action='append', required=True, help='Username; repeat for an owner pool.') + parser.add_argument('--member', action='append', default=[], help='USERNAME:ROLE; repeat for extra members.') + parser.add_argument('--count', type=int, default=10) + parser.add_argument('--seed', type=int, default=42) + parser.add_argument('--batch-label', required=True) + parser.add_argument('--answer-fraction', type=float, nargs=2, default=(0.2, 0.9), metavar=('MIN', 'MAX')) + parser.add_argument('--collection-size', type=int, nargs=2, default=(1, 3), metavar=('MIN', 'MAX')) + parser.add_argument('--created-start', help='ISO datetime with timezone offset.') + parser.add_argument('--created-end', help='ISO datetime with timezone offset.') + + def handle(self, *args, **options): + try: + User = get_user_model() + site = Site.objects.get(pk=options['site_id']) if options['site_id'] else Site.objects.get_current() + catalog = Catalog.objects.get(pk=options['catalog_id']) + owners = [User.objects.get(**{User.USERNAME_FIELD: name}) for name in options['owner']] + members = [] + for member in options['member']: + name, role = member.rsplit(':', 1) + members.append((User.objects.get(**{User.USERNAME_FIELD: name}), role)) + start, end = options['created_start'], options['created_end'] + if bool(start) != bool(end): + raise ValueError('Supply both --created-start and --created-end.') + dates = (datetime.fromisoformat(start), datetime.fromisoformat(end)) if start else None + reports = create_test_projects( + catalog=catalog, site=site, owners=owners, members=members, + count=options['count'], seed=options['seed'], batch_label=options['batch_label'], + answer_fraction=options['answer_fraction'], collection_size=options['collection_size'], + created_between=dates, + ) + except (ObjectDoesNotExist, ValueError, RuntimeError) as exc: + raise CommandError(str(exc)) from exc + self.stdout.write(json.dumps(reports, indent=2)) diff --git a/rdmo_plugins_statistics/management/commands/create_statistics_test_users.py b/rdmo_plugins_statistics/management/commands/create_statistics_test_users.py new file mode 100644 index 0000000..45276c3 --- /dev/null +++ b/rdmo_plugins_statistics/management/commands/create_statistics_test_users.py @@ -0,0 +1,50 @@ +import json +from datetime import datetime + +from django.contrib.sites.models import Site +from django.core.exceptions import ObjectDoesNotExist +from django.core.management.base import BaseCommand, CommandError + +from rdmo_plugins_statistics.testing import create_test_users + + +class Command(BaseCommand): + help = 'Create test users with varied registration dates and RDMO site roles.' + + def add_arguments(self, parser): + parser.add_argument('--site-id', type=int) + parser.add_argument('--count', type=int, default=10) + parser.add_argument('--seed', type=int, default=42) + parser.add_argument('--batch-label', required=True) + parser.add_argument('--joined-start', help='ISO datetime with timezone offset.') + parser.add_argument('--joined-end', help='ISO datetime with timezone offset.') + parser.add_argument('--active-fraction', type=float, default=1) + parser.add_argument('--staff-fraction', type=float, default=0) + parser.add_argument('--manager-fraction', type=float, default=0) + parser.add_argument('--editor-fraction', type=float, default=0) + parser.add_argument('--reviewer-fraction', type=float, default=0) + parser.add_argument('--last-login-fraction', type=float, default=0.7) + + def handle(self, *args, **options): + try: + site = Site.objects.get(pk=options['site_id']) if options['site_id'] else Site.objects.get_current() + start, end = options['joined_start'], options['joined_end'] + if bool(start) != bool(end): + raise ValueError('Supply both --joined-start and --joined-end.') + dates = (datetime.fromisoformat(start), datetime.fromisoformat(end)) if start else None + reports = create_test_users( + site=site, + count=options['count'], + seed=options['seed'], + batch_label=options['batch_label'], + joined_between=dates, + active_fraction=options['active_fraction'], + staff_fraction=options['staff_fraction'], + manager_fraction=options['manager_fraction'], + editor_fraction=options['editor_fraction'], + reviewer_fraction=options['reviewer_fraction'], + last_login_fraction=options['last_login_fraction'], + ) + except (ObjectDoesNotExist, ValueError, RuntimeError) as exc: + raise CommandError(str(exc)) from exc + self.stdout.write(json.dumps(reports, indent=2)) diff --git a/rdmo_plugins_statistics/testing.py b/rdmo_plugins_statistics/testing.py new file mode 100644 index 0000000..417eff4 --- /dev/null +++ b/rdmo_plugins_statistics/testing.py @@ -0,0 +1,335 @@ +"""Create reproducible RDMO development fixtures with Model Bakery.""" + +from datetime import datetime +from decimal import ROUND_FLOOR, Decimal +from random import Random + +from django.contrib.auth import get_user_model +from django.db import transaction +from django.utils.text import slugify + +from rdmo.projects.answers import AnswerTree +from rdmo.projects.models import Membership, Project, Value +from rdmo.projects.progress import compute_progress + + +def _baker(): + try: + from model_bakery import baker + except ImportError as exc: + raise RuntimeError("Install rdmo-plugins-statistics[fixtures] to generate test data.") from exc + return baker + + +def _range(value, minimum, maximum, name): + if not isinstance(value, tuple | list): + value = (value, value) + if len(value) != 2 or not minimum <= value[0] <= value[1] <= maximum: + raise ValueError(f'{name} must be between {minimum} and {maximum}.') + return value + + +def _fraction(value, name): + if not isinstance(value, int | float) or not 0 <= value <= 1: + raise ValueError(f'{name} must be between 0 and 1.') + return value + + +def _datetime_range(value, name): + if value is None: + return None + if not isinstance(value, tuple | list) or len(value) != 2: + raise ValueError(f'{name} requires two timezone-aware datetimes.') + start, end = value + if not all(isinstance(item, datetime) and item.utcoffset() is not None for item in value) or end < start: + raise ValueError(f'{name} requires two ordered timezone-aware datetimes.') + return start, end + + +@transaction.atomic +def create_test_users( + *, + site, + count=10, + seed=42, + batch_label='statistics-users', + joined_between=None, + active_fraction=1, + staff_fraction=0, + manager_fraction=0, + editor_fraction=0, + reviewer_fraction=0, + last_login_fraction=0.7, +): + """Create site members with reproducible account flags and registration dates. + + Passwords are deliberately unusable. Role fractions are independent, so a user + can receive more than one site role. The whole user batch is atomic. + """ + baker = _baker() + User = get_user_model() + if not isinstance(count, int) or count < 1: + raise ValueError('count must be a positive integer.') + label = slugify(batch_label) + if not label: + raise ValueError('batch_label must contain letters or numbers.') + fractions = { + 'active': _fraction(active_fraction, 'active_fraction'), + 'staff': _fraction(staff_fraction, 'staff_fraction'), + 'manager': _fraction(manager_fraction, 'manager_fraction'), + 'editor': _fraction(editor_fraction, 'editor_fraction'), + 'reviewer': _fraction(reviewer_fraction, 'reviewer_fraction'), + 'last_login': _fraction(last_login_fraction, 'last_login_fraction'), + } + dates = _datetime_range(joined_between, 'joined_between') + username_field = User._meta.get_field(User.USERNAME_FIELD) + suffix_length = len(str(count)) + prefix = f'test_{label}_' + if len(prefix) + suffix_length > username_field.max_length: + raise ValueError('batch_label is too long for the configured username field.') + if User.objects.filter(**{f'{User.USERNAME_FIELD}__startswith': prefix}).exists(): + raise ValueError('This user batch label already exists.') + + rng = Random(seed) + reports = [] + for index in range(1, count + 1): + username = f'{prefix}{index:0{suffix_length}d}' + joined = dates[0] + (dates[1] - dates[0]) * rng.random() if dates else None + fields = { + User.USERNAME_FIELD: username, + 'is_active': rng.random() < fractions['active'], + 'is_staff': rng.random() < fractions['staff'], + 'is_superuser': False, + } + if joined is not None: + fields['date_joined'] = joined + if any(field.name == 'email' for field in User._meta.fields): + fields['email'] = f'{username}@example.invalid' + if any(field.name == 'first_name' for field in User._meta.fields): + fields['first_name'] = 'Test' + if any(field.name == 'last_name' for field in User._meta.fields): + fields['last_name'] = f'User {index}' + if joined is not None and rng.random() < fractions['last_login']: + fields['last_login'] = joined + (dates[1] - joined) * rng.random() + else: + fields['last_login'] = None + + user = baker.make(User, **fields) + user.set_unusable_password() + user.save(update_fields=['password']) + user.role.member.add(site) + roles = ['member'] + for role_name in ('manager', 'editor', 'reviewer'): + if rng.random() < fractions[role_name]: + getattr(user.role, role_name).add(site) + roles.append(role_name) + reports.append({ + 'user_id': user.pk, + 'username': user.get_username(), + 'date_joined': user.date_joined.isoformat(), + 'last_login': user.last_login.isoformat() if user.last_login else None, + 'is_active': user.is_active, + 'is_staff': user.is_staff, + 'roles': roles, + }) + return reports + + +def _visible(element, values, location): + conditions = list(element.conditions.all()) + return not conditions or any( + condition.resolve(values, *location) if location else condition.resolve(values) + for condition in conditions + ) + + +def _answer(question, rng, values, location, overrides): + if question.uri in overrides: + payload = dict(overrides[question.uri]) + if payload.keys() - {'text', 'option', 'external_id'}: + raise ValueError(f'Unsupported answer override fields for {question.uri}.') + if payload.get('option') is not None and not question.optionsets.filter(options=payload['option']).exists(): + raise ValueError(f'Option does not belong to {question.uri}.') + return payload, None + + kind = question.value_type + if kind == 'option': + options = [] + for optionset in question.optionsets.all().order_by('pk'): + if not optionset.provider_key and _visible(optionset, values, location): + options.extend(optionset.options.order_by('pk')) + if not options: + return None, 'No eligible static options; supply an answer override.' + option = rng.choice(options) + if option.additional_input: + return None, 'Selected option needs additional input; supply an answer override.' + return {'option': option}, None + if kind in {'integer', 'float'}: + low = Decimal(str(question.minimum if question.minimum is not None else min(0, question.maximum or 0))) + high = Decimal(str(question.maximum if question.maximum is not None else max(low, 100))) + step = Decimal(str(question.step if question.step is not None else 1)) + if not all(x.is_finite() for x in (low, high, step)) or step <= 0 or high < low: + return None, 'Invalid numeric bounds or step.' + if kind == 'integer' and (low != low.to_integral() or step != step.to_integral()): + return None, 'Non-integral integer bounds/step; supply an answer override.' + steps = int(((high - low) / step).to_integral_value(rounding=ROUND_FLOOR)) + number = low + rng.randint(0, steps) * step + return {'text': str(number)}, None + text = { + 'text': f'Test answer {rng.randrange(100000)}', + 'boolean': str(rng.randrange(2)), + 'date': '2026-01-15', + 'datetime': '2026-01-15T12:00:00+00:00', + 'url': 'https://example.org/test', + 'email': 'test@example.org', + 'phone': '+49 30 123456', + }.get(kind) + if text is None: + return None, f'Unsupported value type: {kind}; no files or remote providers are generated.' + return {'text': text}, None + + +@transaction.atomic +def fill_project_interview(project, *, seed=42, answer_fraction=1, collection_size=1, answer_overrides=None): + """Fill an empty interview, retaining hidden answers like the real interview does. + + Overrides map question URIs to Value payloads (text, option, external_id). + Questions are attempted once per location; hidden questions are revisited. + """ + baker = _baker() + fraction = _range(answer_fraction, 0, 1, 'answer_fraction') + sizes = _range(collection_size, 1, 20, 'collection_size') + if any(not isinstance(size, int) for size in sizes): + raise ValueError('collection_size must contain integers.') + if project.catalog is None or project.values.exists(): + raise ValueError('An empty project with a catalog is required.') + rng = Random(seed) + fraction = rng.uniform(*fraction) + overrides = answer_overrides or {} + project.catalog.prefetch_elements() + questions = [e for e in project.catalog.descendants if e._meta.model_name == 'question'] + unknown = overrides.keys() - {q.uri for q in questions} + if unknown: + raise ValueError(f'Unknown question override URIs: {sorted(unknown)}') + attempted, occupied, seen = set(), set(), set() + set_sizes, skipped = {}, {} + + def visit(element, parent=None, parent_collection=False): + kind = element._meta.model_name + values = project.values.filter(snapshot=None) + if kind in {'page', 'questionset', 'question'} and not _visible(element, values, parent): + return + if kind in {'catalog', 'section'}: + for child in element.elements: + visit(child) + elif kind in {'page', 'questionset'}: + key = (kind, element.pk, parent) + if key not in set_sizes: + set_sizes[key] = rng.randint(*sizes) if element.is_collection else 1 + prefix = AnswerTree.compute_child_set_prefix(parent) + if len(prefix) > Value._meta.get_field('set_prefix').max_length: + raise ValueError('Catalog nesting exceeds the Value set_prefix limit.') + for index in range(set_sizes[key]): + for child in element.elements: + visit(child, (prefix, index), element.is_collection) + elif kind == 'question': + seen.add(element.uri) + key = (element.pk, parent) + if key in attempted: + return + attempted.add(key) + if element.attribute_id is None: + skipped[element.uri] = 'Question has no attribute.' + return + if element.uri not in overrides and rng.random() >= fraction: + return + count = rng.randint(*sizes) if element.is_collection else 1 + for index in range(count): + location = (element.attribute_id, *parent, index) + if location in occupied: + continue + payload, reason = _answer(element, rng, values, parent, overrides) + if reason: + skipped[element.uri] = reason + continue + baker.make( + Value, project=project, snapshot=None, attribute=element.attribute, + set_prefix=parent[0], set_index=parent[1], set_collection=parent_collection, + collection_index=index, value_type=element.value_type, unit=element.unit, + **({'text': '', 'option': None, 'external_id': '', 'file': None} | payload), + ) + occupied.add(location) + + for _ in range(32): + before = len(occupied) + visit(project.catalog) + if len(occupied) == before: + break + else: + raise ValueError('Conditional interview did not settle within 32 passes.') + for question in questions: + if question.uri not in seen: + skipped[question.uri] = 'Not visible under the generated answers or ancestor collections.' + project.progress_count, project.progress_total = compute_progress(project) + project.save(update_fields=['progress_count', 'progress_total']) + return { + 'project_id': project.pk, 'value_count': len(occupied), + 'progress_count': project.progress_count, 'progress_total': project.progress_total, + 'skipped': skipped, + } + + +def create_test_projects(*, catalog, site, owners, members=(), count=10, seed=42, + batch_label='statistics-demo', answer_fraction=(0.2, 0.9), + collection_size=(1, 3), created_between=None, answer_overrides=None): + """Create fixtures using existing users/catalog; return one report per project. + + Each project is atomic. Earlier projects remain if a later project fails. + Repeating a batch label is rejected, preventing accidental duplicate runs. + """ + baker = _baker() + owners, members = list(owners), list(members) + if not isinstance(count, int) or count < 1 or not owners: + raise ValueError('Supply a positive count and at least one owner.') + if not batch_label.strip() or len(batch_label) > 180: + raise ValueError('batch_label must contain 1-180 characters.') + if not catalog.available or not catalog.sites.filter(pk=site.pk).exists(): + raise ValueError('Catalog must be available and assigned to the selected site.') + roles = dict(Membership.ROLE_CHOICES) + if any(role not in roles for _, role in members): + raise ValueError('Unknown membership role.') + if len({u.pk for u, _ in members}) != len(members): + raise ValueError('Supply each additional member only once.') + if {u.pk for u in owners} & {u.pk for u, _ in members}: + raise ValueError('Owner pool and additional members must not overlap.') + for user in [*owners, *(u for u, _ in members)]: + if not user.pk or not user.role.member.filter(pk=site.pk).exists(): + raise ValueError(f'User {user} must belong to the selected site.') + _range(answer_fraction, 0, 1, 'answer_fraction') + _range(collection_size, 1, 20, 'collection_size') + dates = _datetime_range(created_between, 'created_between') + prefix = f'[test:{batch_label}] ' + if Project.objects.filter(site=site, title__startswith=prefix).exists(): + raise ValueError('This batch label already exists on the selected site.') + rng, reports = Random(seed), [] + for index in range(count): + with transaction.atomic(): + project = baker.make( + Project, site=site, catalog=catalog, parent=None, + title=f'{prefix}{index + 1:04d}', description='Generated test project.', + progress_count=None, progress_total=None, + ) + baker.make(Membership, project=project, user=owners[index % len(owners)], role='owner') + for user, role in members: + baker.make(Membership, project=project, user=user, role=role) + report = fill_project_interview( + project, seed=rng.randrange(2**32), answer_fraction=answer_fraction, + collection_size=collection_size, answer_overrides=answer_overrides, + ) + if dates: + start, end = dates + created = start + (end - start) * rng.random() + Project.objects.filter(pk=project.pk).update(created=created, updated=created) + project.values.update(created=created, updated=created) + reports.append(report) + return reports diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/settings.py b/tests/settings.py new file mode 100644 index 0000000..26450d8 --- /dev/null +++ b/tests/settings.py @@ -0,0 +1,21 @@ +from rdmo.core.settings import * # noqa: F403 + +INSTALLED_APPS = [ + *INSTALLED_APPS, # noqa: F405 + 'rdmo_plugins_statistics', +] + +ROOT_URLCONF = 'tests.urls' + +SECRET_KEY = 'rdmo-plugins-statistics-tests' + +ALLOWED_HOSTS = ['testserver'] + +STATIC_ROOT = '/tmp/rdmo-plugins-statistics-static' + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': ':memory:', + }, +} diff --git a/tests/test_generator.py b/tests/test_generator.py new file mode 100644 index 0000000..6bb8135 --- /dev/null +++ b/tests/test_generator.py @@ -0,0 +1,147 @@ +import json +from datetime import datetime, timezone +from io import StringIO + +import pytest + +from django.contrib.auth import get_user_model +from django.contrib.sites.models import Site +from django.core.management import call_command + +from rdmo.conditions.models import Condition +from rdmo.domain.models import Attribute +from rdmo.options.models import Option, OptionSet +from rdmo.projects.models import Project +from rdmo.projects.progress import compute_progress +from rdmo.questions.models import Catalog, Page, Question, QuestionSet, Section + +from model_bakery import baker + +from rdmo_plugins_statistics.testing import create_test_projects + +pytestmark = pytest.mark.django_db + + +@pytest.fixture +def fixture_catalog(): + site = Site.objects.get_current() + owner = get_user_model().objects.create_user(username='fixture-owner') + catalog = baker.make(Catalog, uri_prefix='https://example.org', uri_path='catalog', available=True) + catalog.sites.add(site) + section = baker.make(Section, uri_prefix='https://example.org', uri_path='section') + page = baker.make(Page, uri_prefix='https://example.org', uri_path='page', attribute=None, is_collection=False) + catalog.sections.add(section) + section.pages.add(page) + return site, owner, catalog, page + + +def question(parent, name, kind='text', **kwargs): + attribute = baker.make(Attribute, uri_prefix='https://example.org', key=name, parent=None) + item = baker.make( + Question, uri_prefix='https://example.org', uri_path=name, attribute=attribute, + value_type=kind, widget_type='text', is_collection=False, is_optional=False, + minimum=None, maximum=None, step=None, **kwargs, + ) + parent.questions.add(item) + return item + + +def test_projects_memberships_dates_and_progress(fixture_catalog): + site, owner, catalog, page = fixture_catalog + question(page, 'text') + question(page, 'boolean', 'boolean') + guest = get_user_model().objects.create_user(username='fixture-guest') + date = datetime(2025, 1, 1, tzinfo=timezone.utc) + reports = create_test_projects( + catalog=catalog, site=site, owners=[owner], members=[(guest, 'guest')], count=2, + answer_fraction=1, collection_size=1, created_between=(date, date), + ) + assert len(reports) == 2 + for report in reports: + project = Project.objects.get(pk=report['project_id']) + assert project.catalog_id == catalog.pk and project.site_id == site.pk + assert project.created == date + assert set(project.memberships.values_list('user_id', 'role')) == {(owner.pk, 'owner'), (guest.pk, 'guest')} + assert compute_progress(project) == (2, 2) + assert (project.progress_count, project.progress_total) == (2, 2) + assert project.values.filter(snapshot=None).count() == 2 + + +def test_nested_collections_and_conditions(fixture_catalog): + site, owner, catalog, page = fixture_catalog + trigger = question(page, 'trigger') + nested = baker.make(QuestionSet, uri_prefix='https://example.org', uri_path='nested', is_collection=True, attribute=None) + page.questionsets.add(nested) + answer = question(nested, 'nested-answer') + answer.is_collection = True + answer.save() + condition = baker.make( + Condition, uri_prefix='https://example.org', uri_path='condition', source=trigger.attribute, + relation='eq', target_text='show', target_option=None, + ) + nested.conditions.add(condition) + reports = create_test_projects( + catalog=catalog, site=site, owners=[owner], count=1, answer_fraction=1, + collection_size=2, answer_overrides={trigger.uri: {'text': 'show'}}, + ) + project = Project.objects.get(pk=reports[0]['project_id']) + assert set(project.values.filter(attribute=answer.attribute).values_list( + 'set_prefix', 'set_index', 'collection_index', 'set_collection' + )) == {('0', i, j, True) for i in range(2) for j in range(2)} + assert compute_progress(project) == (3, 3) + + +def test_skips_and_reproducibility(fixture_catalog): + site, owner, catalog, page = fixture_catalog + question(page, 'text') + file_question = question(page, 'file', 'file') + outputs = [] + for label in ('one', 'two'): + report = create_test_projects( + catalog=catalog, site=site, owners=[owner], count=1, batch_label=label, + answer_fraction=1, seed=7, + )[0] + assert file_question.uri in report['skipped'] + outputs.append(list(Project.objects.get(pk=report['project_id']).values.values_list('text', flat=True))) + assert outputs[0] == outputs[1] + with pytest.raises(ValueError, match='already exists'): + create_test_projects(catalog=catalog, site=site, owners=[owner], batch_label='one') + assert Project.objects.count() == 2 + + +def test_command_and_validation(fixture_catalog): + site, owner, catalog, page = fixture_catalog + question(page, 'text') + out = StringIO() + call_command('create_statistics_test_projects', catalog_id=catalog.pk, owner=[owner.username], + count=1, batch_label='command', answer_fraction=(0, 0), stdout=out) + report = json.loads(out.getvalue())[0] + assert report['value_count'] == 0 + assert report['progress_total'] == 1 + with pytest.raises(ValueError, match='Unknown question'): + create_test_projects(catalog=catalog, site=site, owners=[owner], answer_overrides={'missing': {}}) + assert Project.objects.count() == 1 + + +def test_numeric_static_options_and_hidden_question(fixture_catalog): + site, owner, catalog, page = fixture_catalog + number = question(page, 'number', 'float') + number.minimum, number.maximum, number.step = 2, 3, 0.25 + number.save() + choice = question(page, 'choice', 'option') + optionset = baker.make(OptionSet, uri_prefix='https://example.org', uri_path='choices', provider_key='') + option = baker.make(Option, uri_prefix='https://example.org', uri_path='only-option', additional_input='') + optionset.options.add(option) + choice.optionsets.add(optionset) + hidden = question(page, 'hidden') + hidden.conditions.add(baker.make( + Condition, uri_prefix='https://example.org', uri_path='never', source=number.attribute, + relation='eq', target_text='999', target_option=None, + )) + report = create_test_projects(catalog=catalog, site=site, owners=[owner], count=1, answer_fraction=1)[0] + project = Project.objects.get(pk=report['project_id']) + assert float(project.values.get(attribute=number.attribute).text) in {2, 2.25, 2.5, 2.75, 3} + assert project.values.get(attribute=choice.attribute).option == option + assert not project.values.filter(attribute=hidden.attribute).exists() + assert hidden.uri in report['skipped'] + assert compute_progress(project) == (2, 2) diff --git a/tests/test_statistics.py b/tests/test_statistics.py new file mode 100644 index 0000000..aa0c132 --- /dev/null +++ b/tests/test_statistics.py @@ -0,0 +1,125 @@ +import pytest + +from django.contrib.auth import get_user_model +from django.contrib.sites.models import Site +from django.urls import reverse +from django.utils import timezone + +from rdmo.projects.models import Project + + +@pytest.mark.django_db +def test_statistics_requires_site_manager(client): + user = get_user_model().objects.create_user(username='member') + client.force_login(user) + + response = client.get(reverse('statistics:index')) + + assert response.status_code == 403 + + +@pytest.mark.django_db +def test_statistics_page_uses_current_site_data(client): + current_site = Site.objects.get_current() + other_site = Site.objects.create(domain='other.example.com', name='Other site') + manager = get_user_model().objects.create_user(username='manager') + manager.role.manager.add(current_site) + + other_user = get_user_model().objects.create_user(username='other-user') + other_user.role.member.remove(current_site) + other_user.role.member.add(other_site) + + project = Project.objects.create( + site=current_site, + title='Private current project', + progress_count=1, + progress_total=2, + ) + Project.objects.create( + site=current_site, + title='Another private current project', + progress_count=1, + progress_total=2, + ) + Project.objects.create( + site=other_site, + title='Private other project', + progress_count=2, + progress_total=2, + ) + client.force_login(manager) + + response = client.get(reverse('statistics:index')) + + assert response.status_code == 200 + project_chart = next(chart for chart in response.context['time_charts'] if chart['key'] == 'project') + user_chart = next(chart for chart in response.context['time_charts'] if chart['key'] == 'user') + catalog_chart = next(chart for chart in response.context['category_charts'] if chart['key'] == 'catalog') + progress_chart = next(chart for chart in response.context['category_charts'] if chart['key'] == 'project-progress') + expected_date = timezone.localtime(project.created).date().isoformat() + + assert project_chart['total'] == 2 + assert project_chart['statistics']['day']['rows'] == [ + {'key': expected_date, 'label': expected_date, 'value': 2}, + ] + assert user_chart['total'] == 1 + assert progress_chart['statistics']['rows'] == [ + {'key': 50, 'label': '50%', 'value': 2}, + ] + assert 'Private current project' not in str(progress_chart) + + +@pytest.mark.django_db +def test_statistics_api_requires_current_site_manager(client): + url = reverse('v1-statistics:statistics-list') + + assert client.get(url).status_code == 401 + + user = get_user_model().objects.create_user(username='api-member') + client.force_login(user) + + assert client.get(url).status_code == 403 + + other_site = Site.objects.create(domain='managed-elsewhere.example.com', name='Managed elsewhere') + user.role.manager.add(other_site) + + assert client.get(url).status_code == 403 + + +@pytest.mark.django_db +def test_statistics_api_returns_current_site_aggregates(client): + current_site = Site.objects.get_current() + other_site = Site.objects.create(domain='api-other.example.com', name='API other site') + manager = get_user_model().objects.create_user(username='api-manager') + manager.role.manager.add(current_site) + Project.objects.create( + site=current_site, + title='Private current project', + progress_count=1, + progress_total=2, + ) + Project.objects.create( + site=other_site, + title='Private other project', + progress_count=2, + progress_total=2, + ) + client.force_login(manager) + url = reverse('v1-statistics:statistics-list') + + response = client.get(url, {'site': other_site.pk}) + + assert response.status_code == 200 + payload = response.json() + assert set(payload) == {'time_charts', 'category_charts'} + project_chart = next(chart for chart in payload['time_charts'] if chart['key'] == 'project') + catalog_chart = next(chart for chart in payload['category_charts'] if chart['key'] == 'catalog') + progress_chart = next(chart for chart in payload['category_charts'] if chart['key'] == 'project-progress') + assert project_chart['total'] == 1 + assert progress_chart['statistics']['rows'] == [ + {'key': 50, 'label': '50%', 'value': 1}, + ] + assert 'query_key' not in project_chart + assert 'date_field' not in project_chart + assert 'Private current project' not in str(payload) + assert client.post(url, data={}).status_code == 405 diff --git a/tests/test_user_generator.py b/tests/test_user_generator.py new file mode 100644 index 0000000..84d56d2 --- /dev/null +++ b/tests/test_user_generator.py @@ -0,0 +1,87 @@ +import json +from datetime import datetime, timezone +from io import StringIO + +import pytest + +from django.contrib.auth import get_user_model +from django.contrib.sites.models import Site +from django.core.management import call_command + +from rdmo_plugins_statistics.testing import create_test_users + +pytestmark = pytest.mark.django_db + + +def test_create_test_users_with_dates_and_roles(): + site = Site.objects.get_current() + start = datetime(2024, 1, 1, tzinfo=timezone.utc) + end = datetime(2025, 1, 1, tzinfo=timezone.utc) + + reports = create_test_users( + site=site, + count=3, + seed=7, + batch_label='dated-users', + joined_between=(start, end), + active_fraction=0, + staff_fraction=1, + manager_fraction=1, + editor_fraction=1, + reviewer_fraction=1, + last_login_fraction=1, + ) + + users = list(get_user_model().objects.filter(username__startswith='test_dated-users_').order_by('username')) + assert len(users) == len(reports) == 3 + assert len({user.date_joined for user in users}) == 3 + for user in users: + assert start <= user.date_joined <= end + assert user.date_joined <= user.last_login <= end + assert not user.is_active + assert user.is_staff + assert not user.has_usable_password() + assert user.role.member.filter(pk=site.pk).exists() + assert user.role.manager.filter(pk=site.pk).exists() + assert user.role.editor.filter(pk=site.pk).exists() + assert user.role.reviewer.filter(pk=site.pk).exists() + + +def test_create_test_users_is_reproducible_and_rejects_duplicate_batch(): + site = Site.objects.get_current() + start = datetime(2024, 1, 1, tzinfo=timezone.utc) + end = datetime(2025, 1, 1, tzinfo=timezone.utc) + first = create_test_users(site=site, count=2, seed=5, batch_label='first', joined_between=(start, end)) + second = create_test_users(site=site, count=2, seed=5, batch_label='second', joined_between=(start, end)) + + assert [item['date_joined'] for item in first] == [item['date_joined'] for item in second] + assert [item['is_active'] for item in first] == [item['is_active'] for item in second] + with pytest.raises(ValueError, match='already exists'): + create_test_users(site=site, batch_label='first') + assert get_user_model().objects.count() == 4 + + +def test_create_statistics_test_users_command(): + out = StringIO() + call_command( + 'create_statistics_test_users', + count=2, + batch_label='command-users', + active_fraction=0, + stdout=out, + ) + reports = json.loads(out.getvalue()) + assert len(reports) == 2 + assert not any(item['is_active'] for item in reports) + assert all(item['roles'] == ['member'] for item in reports) + + +@pytest.mark.parametrize('kwargs', [ + {'count': 0}, + {'active_fraction': 2}, + {'joined_between': (datetime.now(), datetime.now())}, + {'batch_label': '***'}, +]) +def test_create_test_users_validates_input(kwargs): + with pytest.raises(ValueError): + create_test_users(site=Site.objects.get_current(), **kwargs) diff --git a/tests/urls.py b/tests/urls.py new file mode 100644 index 0000000..6308eec --- /dev/null +++ b/tests/urls.py @@ -0,0 +1,11 @@ +from django.urls import include, path + +from rdmo.core.views import home + + +urlpatterns = [ + path('', home, name='home'), + path('', include('rdmo.core.urls')), + path('api/v1/', include('rdmo_plugins_statistics.urls.v1')), + path('statistics/', include('rdmo_plugins_statistics.urls')), +] From 83ccf740125590cc960994554e9a38d0d8b1193f Mon Sep 17 00:00:00 2001 From: David Wallace Date: Wed, 9 Sep 2026 21:27:20 +0200 Subject: [PATCH 04/24] ci: add test job for rdmo versions Signed-off-by: David Wallace --- .github/workflows/ci.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b8c9943..9257b5b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,8 +9,10 @@ on: paths: - '.github/workflows/**' - 'rdmo_plugins_statistics/**' + - 'tests/**' - .pre-commit-config.yaml - pyproject.toml + - README.md # Ref: https://docs.github.com/en/actions/using-jobs/using-concurrency concurrency: @@ -59,11 +61,37 @@ jobs: - run: python -Im pip install -e .[dev] - run: python -Ic 'import rdmo_plugins_statistics; print(rdmo_plugins_statistics.__version__)' + test: + name: "Test (${{ matrix.name }})" + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + include: + - name: RDMO 2.5 + rdmo: "rdmo>=2.5,<3" + - name: RDMO 3.0 release branch + rdmo: "rdmo @ git+https://github.com/rdmorganiser/rdmo.git@3.0.0/release" + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v4 + with: + python-version: "3.12" + cache: pip + - run: python -m pip install --upgrade pip setuptools wheel + - name: Install RDMO target + run: python -m pip install "${{ matrix.rdmo }}" + - name: Install plugin + run: python -m pip install -e '.[dev]' + - name: Run tests + run: pytest + required-checks-pass: if: always() needs: - lint - dev-setup + - test runs-on: ubuntu-22.04 steps: - uses: re-actors/alls-green@release/v1 From c561eb93f158d24e5c4f371eeb21802cbcab6902 Mon Sep 17 00:00:00 2001 From: David Wallace Date: Fri, 11 Sep 2026 09:32:13 +0200 Subject: [PATCH 05/24] Add tests and move baker scripts to tests Signed-off-by: David Wallace --- .../create_statistics_test_projects.py | 52 ------ .../commands/create_statistics_test_users.py | 50 ------ tests/README.md | 120 +++++++++++++ tests/__init__.py | 1 - .../testing.py => tests/helpers.py | 11 +- tests/scripts/__init__.py | 0 tests/scripts/bake_test_projects.py | 63 +++++++ tests/scripts/bake_test_users.py | 55 ++++++ tests/test_generator.py | 14 +- tests/test_statistics.py | 157 ++++++++++++++++-- tests/test_update_chartjs.py | 24 +++ tests/test_user_generator.py | 18 +- tests/urls.py | 1 - 13 files changed, 430 insertions(+), 136 deletions(-) delete mode 100644 rdmo_plugins_statistics/management/commands/create_statistics_test_projects.py delete mode 100644 rdmo_plugins_statistics/management/commands/create_statistics_test_users.py create mode 100644 tests/README.md rename rdmo_plugins_statistics/testing.py => tests/helpers.py (98%) create mode 100644 tests/scripts/__init__.py create mode 100644 tests/scripts/bake_test_projects.py create mode 100644 tests/scripts/bake_test_users.py create mode 100644 tests/test_update_chartjs.py diff --git a/rdmo_plugins_statistics/management/commands/create_statistics_test_projects.py b/rdmo_plugins_statistics/management/commands/create_statistics_test_projects.py deleted file mode 100644 index 2e0a8c8..0000000 --- a/rdmo_plugins_statistics/management/commands/create_statistics_test_projects.py +++ /dev/null @@ -1,52 +0,0 @@ -import json -from datetime import datetime - -from django.contrib.auth import get_user_model -from django.contrib.sites.models import Site -from django.core.exceptions import ObjectDoesNotExist -from django.core.management.base import BaseCommand, CommandError - -from rdmo.questions.models import Catalog - -from rdmo_plugins_statistics.testing import create_test_projects - - -class Command(BaseCommand): - help = 'Create catalog-based test projects, memberships and interview values.' - - def add_arguments(self, parser): - parser.add_argument('--catalog-id', type=int, required=True) - parser.add_argument('--site-id', type=int) - parser.add_argument('--owner', action='append', required=True, help='Username; repeat for an owner pool.') - parser.add_argument('--member', action='append', default=[], help='USERNAME:ROLE; repeat for extra members.') - parser.add_argument('--count', type=int, default=10) - parser.add_argument('--seed', type=int, default=42) - parser.add_argument('--batch-label', required=True) - parser.add_argument('--answer-fraction', type=float, nargs=2, default=(0.2, 0.9), metavar=('MIN', 'MAX')) - parser.add_argument('--collection-size', type=int, nargs=2, default=(1, 3), metavar=('MIN', 'MAX')) - parser.add_argument('--created-start', help='ISO datetime with timezone offset.') - parser.add_argument('--created-end', help='ISO datetime with timezone offset.') - - def handle(self, *args, **options): - try: - User = get_user_model() - site = Site.objects.get(pk=options['site_id']) if options['site_id'] else Site.objects.get_current() - catalog = Catalog.objects.get(pk=options['catalog_id']) - owners = [User.objects.get(**{User.USERNAME_FIELD: name}) for name in options['owner']] - members = [] - for member in options['member']: - name, role = member.rsplit(':', 1) - members.append((User.objects.get(**{User.USERNAME_FIELD: name}), role)) - start, end = options['created_start'], options['created_end'] - if bool(start) != bool(end): - raise ValueError('Supply both --created-start and --created-end.') - dates = (datetime.fromisoformat(start), datetime.fromisoformat(end)) if start else None - reports = create_test_projects( - catalog=catalog, site=site, owners=owners, members=members, - count=options['count'], seed=options['seed'], batch_label=options['batch_label'], - answer_fraction=options['answer_fraction'], collection_size=options['collection_size'], - created_between=dates, - ) - except (ObjectDoesNotExist, ValueError, RuntimeError) as exc: - raise CommandError(str(exc)) from exc - self.stdout.write(json.dumps(reports, indent=2)) diff --git a/rdmo_plugins_statistics/management/commands/create_statistics_test_users.py b/rdmo_plugins_statistics/management/commands/create_statistics_test_users.py deleted file mode 100644 index 45276c3..0000000 --- a/rdmo_plugins_statistics/management/commands/create_statistics_test_users.py +++ /dev/null @@ -1,50 +0,0 @@ -import json -from datetime import datetime - -from django.contrib.sites.models import Site -from django.core.exceptions import ObjectDoesNotExist -from django.core.management.base import BaseCommand, CommandError - -from rdmo_plugins_statistics.testing import create_test_users - - -class Command(BaseCommand): - help = 'Create test users with varied registration dates and RDMO site roles.' - - def add_arguments(self, parser): - parser.add_argument('--site-id', type=int) - parser.add_argument('--count', type=int, default=10) - parser.add_argument('--seed', type=int, default=42) - parser.add_argument('--batch-label', required=True) - parser.add_argument('--joined-start', help='ISO datetime with timezone offset.') - parser.add_argument('--joined-end', help='ISO datetime with timezone offset.') - parser.add_argument('--active-fraction', type=float, default=1) - parser.add_argument('--staff-fraction', type=float, default=0) - parser.add_argument('--manager-fraction', type=float, default=0) - parser.add_argument('--editor-fraction', type=float, default=0) - parser.add_argument('--reviewer-fraction', type=float, default=0) - parser.add_argument('--last-login-fraction', type=float, default=0.7) - - def handle(self, *args, **options): - try: - site = Site.objects.get(pk=options['site_id']) if options['site_id'] else Site.objects.get_current() - start, end = options['joined_start'], options['joined_end'] - if bool(start) != bool(end): - raise ValueError('Supply both --joined-start and --joined-end.') - dates = (datetime.fromisoformat(start), datetime.fromisoformat(end)) if start else None - reports = create_test_users( - site=site, - count=options['count'], - seed=options['seed'], - batch_label=options['batch_label'], - joined_between=dates, - active_fraction=options['active_fraction'], - staff_fraction=options['staff_fraction'], - manager_fraction=options['manager_fraction'], - editor_fraction=options['editor_fraction'], - reviewer_fraction=options['reviewer_fraction'], - last_login_fraction=options['last_login_fraction'], - ) - except (ObjectDoesNotExist, ValueError, RuntimeError) as exc: - raise CommandError(str(exc)) from exc - self.stdout.write(json.dumps(reports, indent=2)) diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..b0599d6 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,120 @@ +# Testing and development fixtures + +Install the development dependencies from a source checkout: + +```bash +pip install -e '.[dev]' +``` + +### Test users + +Create site members with registration and last-login dates distributed across a +time range: + +```bash +python manage.py runscript tests.scripts.bake_test_users \ + --script-args="--count 50 --seed 42 --batch-label september-users \ + --joined-start 2024-01-01T00:00:00+00:00 \ + --joined-end 2026-09-01T00:00:00+00:00 \ + --active-fraction 0.9 --last-login-fraction 0.7 \ + --manager-fraction 0.1 --editor-fraction 0.2" +``` + +The fractions independently control active accounts, staff status, last logins, +and RDMO manager, editor, and reviewer roles. Every generated user is a member of +the selected site. Use `--site-id` to select another site; otherwise the configured +current site is used. RDMO may additionally assign its current site through its +normal user-creation signal. + +Usernames follow `test_BATCH_NUMBER`, email addresses use the reserved +`example.invalid` domain, and passwords are unusable. Existing usernames with the +same normalized batch prefix cause the script to stop before creating anything. +The script writes a JSON report containing IDs, usernames, dates, flags, and roles. +The whole user batch is created in one transaction. + +The reusable function accepts the same controls: + +```python +from tests.helpers import create_test_users + +users = create_test_users( + site=site, + count=50, + batch_label='september-users', + seed=42, + joined_between=(start_datetime, end_datetime), + active_fraction=0.9, + staff_fraction=0.05, + manager_fraction=0.1, + editor_fraction=0.2, + reviewer_fraction=0.1, + last_login_fraction=0.7, +) +``` + +### Test projects + +The script uses an existing catalog and existing users belonging to +the selected site. It creates projects, explicit memberships, and current interview +values, then calculates progress with RDMO's answer tree: + +```bash +python manage.py runscript tests.scripts.bake_test_projects \ + --script-args="--catalog-id 12 --owner alice --member bob:author \ + --count 100 --seed 42 --batch-label september-demo \ + --answer-fraction 0.2 0.9 --collection-size 1 3 \ + --created-start 2025-01-01T00:00:00+00:00 \ + --created-end 2026-09-01T00:00:00+00:00" +``` + +Use `--site-id` to select another site; otherwise the configured current site is +used. Repeat `--owner` to distribute ownership across users. Repeat `--member` +to assign additional `owner`, `manager`, `author`, or `guest` memberships. Owners +and additional members must not overlap. No users or catalogs are created. + +The same implementation is available from Django shell or pytest: + +```python +from tests.helpers import create_test_projects + +reports = create_test_projects( + catalog=catalog, + site=site, + owners=[alice], + members=[(bob, 'author')], + count=10, + batch_label='interview-demo', + seed=42, + answer_fraction=1, + collection_size=(1, 2), + answer_overrides={ + # Keys are question URIs; values are Value field dictionaries. + trigger_question.uri: {'text': 'yes'}, + choice_question.uri: {'option': option}, + dynamic_question.uri: {'text': 'Fixture label', 'external_id': 'fixture:1'}, + }, +) +``` + +`fill_project_interview(project, ...)` also fills an existing empty project. +Overrides support `text`, `option`, and `external_id`, bypass automatic answer +selection, and are applied only when the question is visible. Supply semantically +valid payloads; overrides are not an interview-form validation API. + +Ordinary scalar answers, static options, repeated pages/question sets, and repeated +questions are supported. Conditions are evaluated using RDMO and revisited for up +to 32 passes as answers reveal questions. Answers subsequently hidden by conditions +remain stored, as in a normal interview. File questions, dynamic-only option sets, +and options requiring additional input are reported as skipped unless a supported +explicit payload is supplied. No remote option providers are called or files created. + +Answer fraction is a probability for attempting each eligible question location, +not a guaranteed progress percentage. JSON reports contain project IDs, value +counts, actual progress counts/totals, and skipped-question reasons. A seed makes +generated answer content reproducible for identical inputs/catalogs; IDs and +timestamps are not deterministic unless an explicit date range is provided. + +Projects are labelled `[test:BATCH]`; an existing batch label on the site is +rejected. Each project is created in a transaction. If a later project fails, +earlier completed projects remain and can be identified by that batch label. +The script does not overwrite or delete existing projects. diff --git a/tests/__init__.py b/tests/__init__.py index 8b13789..e69de29 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1 +0,0 @@ - diff --git a/rdmo_plugins_statistics/testing.py b/tests/helpers.py similarity index 98% rename from rdmo_plugins_statistics/testing.py rename to tests/helpers.py index 417eff4..83030ad 100644 --- a/rdmo_plugins_statistics/testing.py +++ b/tests/helpers.py @@ -12,13 +12,7 @@ from rdmo.projects.models import Membership, Project, Value from rdmo.projects.progress import compute_progress - -def _baker(): - try: - from model_bakery import baker - except ImportError as exc: - raise RuntimeError("Install rdmo-plugins-statistics[fixtures] to generate test data.") from exc - return baker +from model_bakery import baker def _range(value, minimum, maximum, name): @@ -66,7 +60,6 @@ def create_test_users( Passwords are deliberately unusable. Role fractions are independent, so a user can receive more than one site role. The whole user batch is atomic. """ - baker = _baker() User = get_user_model() if not isinstance(count, int) or count < 1: raise ValueError('count must be a positive integer.') @@ -196,7 +189,6 @@ def fill_project_interview(project, *, seed=42, answer_fraction=1, collection_si Overrides map question URIs to Value payloads (text, option, external_id). Questions are attempted once per location; hidden questions are revisited. """ - baker = _baker() fraction = _range(answer_fraction, 0, 1, 'answer_fraction') sizes = _range(collection_size, 1, 20, 'collection_size') if any(not isinstance(size, int) for size in sizes): @@ -287,7 +279,6 @@ def create_test_projects(*, catalog, site, owners, members=(), count=10, seed=42 Each project is atomic. Earlier projects remain if a later project fails. Repeating a batch label is rejected, preventing accidental duplicate runs. """ - baker = _baker() owners, members = list(owners), list(members) if not isinstance(count, int) or count < 1 or not owners: raise ValueError('Supply a positive count and at least one owner.') diff --git a/tests/scripts/__init__.py b/tests/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/scripts/bake_test_projects.py b/tests/scripts/bake_test_projects.py new file mode 100644 index 0000000..fb2f42f --- /dev/null +++ b/tests/scripts/bake_test_projects.py @@ -0,0 +1,63 @@ +import argparse +import json +import shlex +from datetime import datetime + +from django.contrib.auth import get_user_model +from django.contrib.sites.models import Site +from django.core.exceptions import ObjectDoesNotExist +from django.core.management.base import CommandError + +from rdmo.questions.models import Catalog + +from tests.helpers import create_test_projects + + +def get_arguments(args): + parser = argparse.ArgumentParser(description='Create catalog-based test projects and interview values.') + parser.add_argument('--catalog-id', type=int, required=True) + parser.add_argument('--site-id', type=int) + parser.add_argument('--owner', action='append', required=True, help='Username; repeat for an owner pool.') + parser.add_argument('--member', action='append', default=[], help='USERNAME:ROLE; repeat for extra members.') + parser.add_argument('--count', type=int, default=10) + parser.add_argument('--seed', type=int, default=42) + parser.add_argument('--batch-label', required=True) + parser.add_argument('--answer-fraction', type=float, nargs=2, default=(0.2, 0.9), metavar=('MIN', 'MAX')) + parser.add_argument('--collection-size', type=int, nargs=2, default=(1, 3), metavar=('MIN', 'MAX')) + parser.add_argument('--created-start', help='ISO datetime with timezone offset.') + parser.add_argument('--created-end', help='ISO datetime with timezone offset.') + return parser.parse_args(shlex.split(' '.join(args))) + + +def run(*args): + options = get_arguments(args) + try: + User = get_user_model() + site = Site.objects.get(pk=options.site_id) if options.site_id else Site.objects.get_current() + catalog = Catalog.objects.get(pk=options.catalog_id) + owners = [User.objects.get(**{User.USERNAME_FIELD: name}) for name in options.owner] + members = [] + for member in options.member: + name, role = member.rsplit(':', 1) + members.append((User.objects.get(**{User.USERNAME_FIELD: name}), role)) + if bool(options.created_start) != bool(options.created_end): + raise ValueError('Supply both --created-start and --created-end.') + dates = ( + (datetime.fromisoformat(options.created_start), datetime.fromisoformat(options.created_end)) + if options.created_start else None + ) + reports = create_test_projects( + catalog=catalog, + site=site, + owners=owners, + members=members, + count=options.count, + seed=options.seed, + batch_label=options.batch_label, + answer_fraction=options.answer_fraction, + collection_size=options.collection_size, + created_between=dates, + ) + except (ObjectDoesNotExist, ValueError) as exc: + raise CommandError(str(exc)) from exc + print(json.dumps(reports, indent=2)) diff --git a/tests/scripts/bake_test_users.py b/tests/scripts/bake_test_users.py new file mode 100644 index 0000000..7f51388 --- /dev/null +++ b/tests/scripts/bake_test_users.py @@ -0,0 +1,55 @@ +import argparse +import json +import shlex +from datetime import datetime + +from django.contrib.sites.models import Site +from django.core.exceptions import ObjectDoesNotExist +from django.core.management.base import CommandError + +from tests.helpers import create_test_users + + +def get_arguments(args): + parser = argparse.ArgumentParser(description='Create test users with varied dates and RDMO site roles.') + parser.add_argument('--site-id', type=int) + parser.add_argument('--count', type=int, default=10) + parser.add_argument('--seed', type=int, default=42) + parser.add_argument('--batch-label', required=True) + parser.add_argument('--joined-start', help='ISO datetime with timezone offset.') + parser.add_argument('--joined-end', help='ISO datetime with timezone offset.') + parser.add_argument('--active-fraction', type=float, default=1) + parser.add_argument('--staff-fraction', type=float, default=0) + parser.add_argument('--manager-fraction', type=float, default=0) + parser.add_argument('--editor-fraction', type=float, default=0) + parser.add_argument('--reviewer-fraction', type=float, default=0) + parser.add_argument('--last-login-fraction', type=float, default=0.7) + return parser.parse_args(shlex.split(' '.join(args))) + + +def run(*args): + options = get_arguments(args) + try: + site = Site.objects.get(pk=options.site_id) if options.site_id else Site.objects.get_current() + if bool(options.joined_start) != bool(options.joined_end): + raise ValueError('Supply both --joined-start and --joined-end.') + dates = ( + (datetime.fromisoformat(options.joined_start), datetime.fromisoformat(options.joined_end)) + if options.joined_start else None + ) + reports = create_test_users( + site=site, + count=options.count, + seed=options.seed, + batch_label=options.batch_label, + joined_between=dates, + active_fraction=options.active_fraction, + staff_fraction=options.staff_fraction, + manager_fraction=options.manager_fraction, + editor_fraction=options.editor_fraction, + reviewer_fraction=options.reviewer_fraction, + last_login_fraction=options.last_login_fraction, + ) + except (ObjectDoesNotExist, ValueError) as exc: + raise CommandError(str(exc)) from exc + print(json.dumps(reports, indent=2)) diff --git a/tests/test_generator.py b/tests/test_generator.py index 6bb8135..9298eae 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -1,4 +1,5 @@ import json +from contextlib import redirect_stdout from datetime import datetime, timezone from io import StringIO @@ -17,7 +18,7 @@ from model_bakery import baker -from rdmo_plugins_statistics.testing import create_test_projects +from tests.helpers import create_test_projects pytestmark = pytest.mark.django_db @@ -113,8 +114,15 @@ def test_command_and_validation(fixture_catalog): site, owner, catalog, page = fixture_catalog question(page, 'text') out = StringIO() - call_command('create_statistics_test_projects', catalog_id=catalog.pk, owner=[owner.username], - count=1, batch_label='command', answer_fraction=(0, 0), stdout=out) + with redirect_stdout(out): + call_command( + 'runscript', + 'tests.scripts.bake_test_projects', + script_args=[ + f'--catalog-id {catalog.pk} --owner {owner.username} ' + '--count 1 --batch-label command --answer-fraction 0 0' + ], + ) report = json.loads(out.getvalue())[0] assert report['value_count'] == 0 assert report['progress_total'] == 1 diff --git a/tests/test_statistics.py b/tests/test_statistics.py index aa0c132..290e9bb 100644 --- a/tests/test_statistics.py +++ b/tests/test_statistics.py @@ -1,11 +1,23 @@ +import csv + import pytest from django.contrib.auth import get_user_model from django.contrib.sites.models import Site +from django.core.management import call_command +from django.core.management.base import CommandError from django.urls import reverse from django.utils import timezone from rdmo.projects.models import Project +from rdmo.questions.models import Catalog + +from rdmo_plugins_statistics.statistics import ( + fetch_catalog_statistics_for_sites, + fetch_project_statistics_for_sites, + fetch_statistics_for_sites, + fetch_user_statistics_for_sites, +) @pytest.mark.django_db @@ -63,6 +75,7 @@ def test_statistics_page_uses_current_site_data(client): {'key': expected_date, 'label': expected_date, 'value': 2}, ] assert user_chart['total'] == 1 + assert set(catalog_chart['statistics']) == {'rows'} assert progress_chart['statistics']['rows'] == [ {'key': 50, 'label': '50%', 'value': 2}, ] @@ -86,6 +99,26 @@ def test_statistics_api_requires_current_site_manager(client): assert client.get(url).status_code == 403 +@pytest.mark.parametrize( + 'url_name', + ( + 'v1-statistics:project-statistics-list', + 'v1-statistics:user-statistics-list', + 'v1-statistics:catalog-statistics-list', + ), +) +@pytest.mark.django_db +def test_domain_statistics_apis_require_current_site_manager(client, url_name): + url = reverse(url_name) + + assert client.get(url).status_code == 401 + + user = get_user_model().objects.create_user(username=f'{url_name}-member') + client.force_login(user) + + assert client.get(url).status_code == 403 + + @pytest.mark.django_db def test_statistics_api_returns_current_site_aggregates(client): current_site = Site.objects.get_current() @@ -111,15 +144,119 @@ def test_statistics_api_returns_current_site_aggregates(client): assert response.status_code == 200 payload = response.json() - assert set(payload) == {'time_charts', 'category_charts'} - project_chart = next(chart for chart in payload['time_charts'] if chart['key'] == 'project') - catalog_chart = next(chart for chart in payload['category_charts'] if chart['key'] == 'catalog') - progress_chart = next(chart for chart in payload['category_charts'] if chart['key'] == 'project-progress') - assert project_chart['total'] == 1 - assert progress_chart['statistics']['rows'] == [ - {'key': 50, 'label': '50%', 'value': 1}, - ] - assert 'query_key' not in project_chart - assert 'date_field' not in project_chart + assert set(payload) == {'projects', 'users', 'catalogs'} + assert payload['projects']['total'] == 1 + assert payload['projects']['progress'] == [{'percentage': 50, 'count': 1}] + assert payload['catalogs'] == {'usage': []} assert 'Private current project' not in str(payload) assert client.post(url, data={}).status_code == 405 + + +@pytest.mark.django_db +def test_domain_statistics_apis_compose_the_aggregate_response(client): + current_site = Site.objects.get_current() + manager = get_user_model().objects.create_user(username='domain-api-manager') + manager.role.manager.add(current_site) + Project.objects.create( + site=current_site, + title='Domain API project', + progress_count=1, + progress_total=4, + ) + client.force_login(manager) + + aggregate = client.get(reverse('v1-statistics:statistics-list')).json() + projects = client.get(reverse('v1-statistics:project-statistics-list')).json() + users = client.get(reverse('v1-statistics:user-statistics-list')).json() + catalogs = client.get(reverse('v1-statistics:catalog-statistics-list')).json() + + assert set(projects) == {'total', 'created', 'progress'} + assert set(users) == {'total', 'registered'} + assert set(catalogs) == {'usage'} + assert aggregate == {'projects': projects, 'users': users, 'catalogs': catalogs} + + +@pytest.mark.django_db +def test_statistics_can_be_fetched_for_multiple_sites(client, tmp_path): + current_site = Site.objects.get_current() + other_site = Site.objects.create(domain='statistics-other.example.com', name='Statistics other site') + + current_user = get_user_model().objects.create_user(username='statistics-current-user') + current_user.role.member.add(current_site) + other_user = get_user_model().objects.create_user(username='statistics-other-user') + other_user.role.member.remove(current_site) + other_user.role.member.add(other_site) + shared_user = get_user_model().objects.create_user(username='shared-user') + shared_user.role.member.set([current_site, other_site]) + current_user.role.member.set([current_site]) + extra_user = get_user_model().objects.create_user(username='extra-user') + extra_user.role.member.set([other_site]) + + catalog = Catalog.objects.create(uri_prefix='https://example.org', uri_path='catalog', title_lang1='=Catalog') + catalog.sites.set([current_site, other_site]) + Project.objects.create(site=current_site, catalog=catalog, title='Current site project') + Project.objects.create(site=other_site, catalog=catalog, title='Other site project') + Project.objects.create( + site=other_site, catalog=catalog, title='Second other project', + progress_count=1, progress_total=2, + ) + + statistics = fetch_statistics_for_sites((current_site, other_site)) + + assert [entry['site'] for entry in statistics] == [ + { + 'id': current_site.pk, + 'name': current_site.name, + 'domain': current_site.domain, + }, + { + 'id': other_site.pk, + 'name': other_site.name, + 'domain': other_site.domain, + }, + ] + assert [entry['projects']['total'] for entry in statistics] == [1, 2] + assert [entry['users']['total'] for entry in statistics] == [2, 3] + assert [entry['catalogs']['usage'][0]['project_count'] for entry in statistics] == [1, 2] + for fetcher, domain in ( + (fetch_project_statistics_for_sites, 'projects'), + (fetch_user_statistics_for_sites, 'users'), + (fetch_catalog_statistics_for_sites, 'catalogs'), + ): + assert fetcher((current_site, other_site)) == [ + {'site': entry['site'], **entry[domain]} for entry in statistics + ] + assert fetch_project_statistics_for_sites(()) == [] + assert fetch_user_statistics_for_sites(()) == [] + assert fetch_catalog_statistics_for_sites(()) == [] + + current_user.role.manager.add(current_site) + client.force_login(current_user) + response = client.get(reverse('statistics:index')) + assert response.status_code == 200 + assert response.context['time_charts'][0]['total'] == 1 + assert response.context['category_charts'][0]['statistics']['rows'][0]['value'] == 1 + + call_command('export_statistics', site_id=[current_site.pk, other_site.pk], output_dir=tmp_path) + + def read_table(name): + with (tmp_path / name).open(newline='', encoding='utf-8') as stream: + return list(csv.DictReader(stream)) + + assert [row['project_count'] for row in read_table('site_totals.csv')] == ['1', '2'] + assert [row['user_count'] for row in read_table('site_totals.csv')] == ['2', '3'] + assert [row['project_count'] for row in read_table('project_creation.csv')] == ['1', '2'] + assert [row['user_count'] for row in read_table('user_registration.csv')] == ['2', '3'] + assert [row['percentage'] for row in read_table('project_progress.csv')] == ['0', '0', '50'] + assert [row['catalog_title'] for row in read_table('catalog_usage.csv')] == ["'=Catalog", "'=Catalog"] + + empty_site = Site.objects.create(domain='empty.example.org', name='Empty') + call_command('export_statistics', site_id=[empty_site.pk], output_dir=tmp_path) + assert len(read_table('site_totals.csv')) == 1 + for name in ('project_creation.csv', 'user_registration.csv', 'project_progress.csv', 'catalog_usage.csv'): + assert read_table(name) == [] + assert (tmp_path / name).read_text().startswith('site_id,site_name,site_domain,') + previous = (tmp_path / 'site_totals.csv').read_bytes() + with pytest.raises(CommandError, match='Unknown site IDs'): + call_command('export_statistics', site_id=[empty_site.pk, 999999], output_dir=tmp_path) + assert (tmp_path / 'site_totals.csv').read_bytes() == previous diff --git a/tests/test_update_chartjs.py b/tests/test_update_chartjs.py new file mode 100644 index 0000000..cc66d81 --- /dev/null +++ b/tests/test_update_chartjs.py @@ -0,0 +1,24 @@ +import pytest + +from scripts import update_chartjs + + +def test_update_chartjs_replaces_only_valid_downloads(monkeypatch, tmp_path): + javascript = b'/*! Chart.js v4.5.1 | Released under the MIT License */' + license_text = b'The MIT License (MIT)\nChart.js Contributors\n' + downloads = { + 'https://cdn.jsdelivr.net/npm/chart.js@4.5.1/dist/chart.umd.min.js': javascript, + 'https://cdn.jsdelivr.net/npm/chart.js@4.5.1/LICENSE.md': license_text, + } + monkeypatch.setattr(update_chartjs, 'download', downloads.__getitem__) + + update_chartjs.update_chartjs('4.5.1', tmp_path) + + assert (tmp_path / 'chart.umd.min.js').read_bytes() == javascript + assert (tmp_path / 'chart.LICENSE.md').read_bytes() == license_text + + monkeypatch.setattr(update_chartjs, 'download', lambda url: b'invalid') + with pytest.raises(ValueError, match='JavaScript does not match'): + update_chartjs.update_chartjs('4.5.2', tmp_path) + assert (tmp_path / 'chart.umd.min.js').read_bytes() == javascript + assert (tmp_path / 'chart.LICENSE.md').read_bytes() == license_text diff --git a/tests/test_user_generator.py b/tests/test_user_generator.py index 84d56d2..88d6956 100644 --- a/tests/test_user_generator.py +++ b/tests/test_user_generator.py @@ -1,4 +1,5 @@ import json +from contextlib import redirect_stdout from datetime import datetime, timezone from io import StringIO @@ -8,7 +9,7 @@ from django.contrib.sites.models import Site from django.core.management import call_command -from rdmo_plugins_statistics.testing import create_test_users +from tests.helpers import create_test_users pytestmark = pytest.mark.django_db @@ -61,15 +62,14 @@ def test_create_test_users_is_reproducible_and_rejects_duplicate_batch(): assert get_user_model().objects.count() == 4 -def test_create_statistics_test_users_command(): +def test_bake_test_users_script(): out = StringIO() - call_command( - 'create_statistics_test_users', - count=2, - batch_label='command-users', - active_fraction=0, - stdout=out, - ) + with redirect_stdout(out): + call_command( + 'runscript', + 'tests.scripts.bake_test_users', + script_args=['--count 2 --batch-label command-users --active-fraction 0'], + ) reports = json.loads(out.getvalue()) assert len(reports) == 2 assert not any(item['is_active'] for item in reports) diff --git a/tests/urls.py b/tests/urls.py index 6308eec..de6337b 100644 --- a/tests/urls.py +++ b/tests/urls.py @@ -2,7 +2,6 @@ from rdmo.core.views import home - urlpatterns = [ path('', home, name='home'), path('', include('rdmo.core.urls')), From 566d7f21a975b7fa35dbdf382e8cfd9e7b0047d9 Mon Sep 17 00:00:00 2001 From: David Wallace Date: Fri, 11 Sep 2026 09:37:16 +0200 Subject: [PATCH 06/24] build: use hatchling for build and vcs for version Signed-off-by: David Wallace --- .github/workflows/ci.yml | 13 +++++++++++-- pyproject.toml | 17 +++++++---------- rdmo_plugins_statistics/__init__.py | 8 +++++++- 3 files changed, 25 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9257b5b..c08551c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,6 +9,7 @@ on: paths: - '.github/workflows/**' - 'rdmo_plugins_statistics/**' + - 'scripts/**' - 'tests/**' - .pre-commit-config.yaml - pyproject.toml @@ -35,7 +36,7 @@ jobs: with: python-version: "3.12" cache: pip - - run: python -m pip install --upgrade pip setuptools wheel + - run: python -m pip install --upgrade pip - run: python -m pip install -e .[dev] - name: Set up pre-commit cache uses: actions/cache@v3 @@ -78,13 +79,21 @@ jobs: with: python-version: "3.12" cache: pip - - run: python -m pip install --upgrade pip setuptools wheel + - run: python -m pip install --upgrade pip - name: Install RDMO target run: python -m pip install "${{ matrix.rdmo }}" - name: Install plugin run: python -m pip install -e '.[dev]' - name: Run tests run: pytest + - name: Build wheel + run: python -m pip wheel --no-deps . --wheel-dir dist + - name: Verify wheel assets + run: | + unzip -l dist/*.whl | grep -q 'rdmo_plugins_statistics/templates/rdmo_plugins_statistics/statistics.html' + unzip -l dist/*.whl | grep -q 'rdmo_plugins_statistics/locale/de/LC_MESSAGES/django.mo' + unzip -l dist/*.whl | grep -q 'rdmo_plugins_statistics/static/rdmo_plugins_statistics/css/statistics.css' + unzip -l dist/*.whl | grep -q 'rdmo_plugins_statistics/static/rdmo_plugins_statistics/js/chart.umd.min.js' required-checks-pass: if: always() diff --git a/pyproject.toml b/pyproject.toml index e249b5d..2a23f4e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [build-system] -build-backend = "setuptools.build_meta" -requires = ["setuptools", "wheel"] +build-backend = "hatchling.build" +requires = ["hatchling", "hatch-vcs"] [project] name = "rdmo-plugins-statistics" @@ -41,7 +41,6 @@ dependencies = [ ] [project.optional-dependencies] -fixtures = ["model-bakery>=1.20,<2"] dev = [ "model-bakery>=1.20,<2", "pre-commit", @@ -62,14 +61,12 @@ issues = "https://github.com/rdmorganiser/rdmo-plugins-statistics/issues" repository = "https://github.com/rdmorganiser/rdmo-plugins-statistics.git" slack = "https://rdmo.slack.com" -[tool.setuptools.packages.find] -include = ["rdmo_plugins_statistics*"] +[tool.hatch.version] +source = "vcs" +fallback-version = "0.0.1" -[tool.setuptools.package-data] -"*" = ["*"] - -[tool.setuptools.dynamic] -version = {attr = "rdmo_plugins_statistics.__version__"} +[tool.hatch.build.targets.wheel] +packages = ["rdmo_plugins_statistics"] [tool.ruff] target-version = "py310" diff --git a/rdmo_plugins_statistics/__init__.py b/rdmo_plugins_statistics/__init__.py index f102a9c..7776efc 100644 --- a/rdmo_plugins_statistics/__init__.py +++ b/rdmo_plugins_statistics/__init__.py @@ -1 +1,7 @@ -__version__ = "0.0.1" +from importlib.metadata import PackageNotFoundError +from importlib.metadata import version as _version + +try: + __version__ = _version(__package__) +except PackageNotFoundError: + __version__ = "0.0.0+unknown" From 040364b182349df4f506dac0d4cb692935e155e7 Mon Sep 17 00:00:00 2001 From: David Wallace Date: Fri, 11 Sep 2026 09:40:45 +0200 Subject: [PATCH 07/24] Add chartjs update script, license and keep stable name Signed-off-by: David Wallace --- .../js/chart.LICENSE.md | 9 +++ ...hart-4.5.1.umd.min.js => chart.umd.min.js} | 0 .../rdmo_plugins_statistics/statistics.html | 2 +- scripts/__init__.py | 0 scripts/update_chartjs.py | 61 +++++++++++++++++++ 5 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 rdmo_plugins_statistics/static/rdmo_plugins_statistics/js/chart.LICENSE.md rename rdmo_plugins_statistics/static/rdmo_plugins_statistics/js/{chart-4.5.1.umd.min.js => chart.umd.min.js} (100%) create mode 100644 scripts/__init__.py create mode 100644 scripts/update_chartjs.py diff --git a/rdmo_plugins_statistics/static/rdmo_plugins_statistics/js/chart.LICENSE.md b/rdmo_plugins_statistics/static/rdmo_plugins_statistics/js/chart.LICENSE.md new file mode 100644 index 0000000..f216610 --- /dev/null +++ b/rdmo_plugins_statistics/static/rdmo_plugins_statistics/js/chart.LICENSE.md @@ -0,0 +1,9 @@ +The MIT License (MIT) + +Copyright (c) 2014-2024 Chart.js Contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/rdmo_plugins_statistics/static/rdmo_plugins_statistics/js/chart-4.5.1.umd.min.js b/rdmo_plugins_statistics/static/rdmo_plugins_statistics/js/chart.umd.min.js similarity index 100% rename from rdmo_plugins_statistics/static/rdmo_plugins_statistics/js/chart-4.5.1.umd.min.js rename to rdmo_plugins_statistics/static/rdmo_plugins_statistics/js/chart.umd.min.js diff --git a/rdmo_plugins_statistics/templates/rdmo_plugins_statistics/statistics.html b/rdmo_plugins_statistics/templates/rdmo_plugins_statistics/statistics.html index c444d24..185c102 100644 --- a/rdmo_plugins_statistics/templates/rdmo_plugins_statistics/statistics.html +++ b/rdmo_plugins_statistics/templates/rdmo_plugins_statistics/statistics.html @@ -189,7 +189,7 @@

{{ chart.title }}

{{ block.super }} - + + {% endblock %} diff --git a/tests/test_statistics.py b/tests/test_statistics.py index 0327843..5a48517 100644 --- a/tests/test_statistics.py +++ b/tests/test_statistics.py @@ -13,11 +13,64 @@ from rdmo.projects.models import Project from rdmo.questions.models import Catalog +from rdmo_plugins_statistics.charts import compute_dashboard_summary, compute_project_progress_statistics +from rdmo_plugins_statistics.config import CATEGORY_CHART_DEFINITION from rdmo_plugins_statistics.statistics import ( fetch_statistics_for_sites, ) +def test_dashboard_summary_uses_existing_aggregates(): + statistics = { + 'projects': { + 'total': 8, + 'progress': [ + {'percentage': 50, 'count': 3}, + {'percentage': 100, 'count': 2}, + ], + }, + 'users': {'total': 5}, + 'catalogs': { + 'usage': [ + {'project_count': 4}, + {'project_count': 0}, + {'project_count': 1}, + ], + }, + } + + assert compute_dashboard_summary(statistics) == { + 'projects': 8, + 'users': 5, + 'catalogs_in_use': 2, + 'complete_projects': 2, + } + + statistics['projects']['progress'] = [{'percentage': 50, 'count': 3}] + assert compute_dashboard_summary(statistics)['complete_projects'] == 0 + + +def test_project_progress_statistics_uses_the_configured_group_size(): + statistics = [ + {'percentage': 0, 'count': 1}, + {'percentage': 4, 'count': 2}, + {'percentage': 5, 'count': 3}, + {'percentage': 99, 'count': 4}, + {'percentage': 100, 'count': 5}, + ] + + rows = compute_project_progress_statistics( + statistics, + CATEGORY_CHART_DEFINITION['project_progress']['progress_group_size'], + )['rows'] + + assert len(rows) == 11 + assert rows[0] == {'key': 0, 'label': '0-9%', 'value': 6} + assert rows[1] == {'key': 10, 'label': '10-19%', 'value': 0} + assert rows[9] == {'key': 90, 'label': '90-99%', 'value': 4} + assert rows[10] == {'key': 100, 'label': '100%', 'value': 5} + + @pytest.mark.django_db def test_statistics_requires_site_manager(client): user = get_user_model().objects.create_user(username='member') @@ -51,12 +104,22 @@ def test_statistics_page_uses_current_site_data(client): progress_count=1, progress_total=2, ) - Project.objects.create( + other_project = Project.objects.create( site=other_site, title='Private other project', progress_count=2, progress_total=2, ) + catalog = Catalog.objects.create( + uri_prefix='https://example.org', + uri_path='summary-catalog', + title_lang1='Summary catalog', + ) + catalog.sites.set([current_site, other_site]) + project.catalog = catalog + project.save(update_fields=['catalog']) + other_project.catalog = catalog + other_project.save(update_fields=['catalog']) client.force_login(manager) response = client.get(reverse('statistics:index')) @@ -64,32 +127,75 @@ def test_statistics_page_uses_current_site_data(client): assert response.status_code == 200 project_chart = next(chart for chart in response.context['time_charts'] if chart['key'] == 'project') user_chart = next(chart for chart in response.context['time_charts'] if chart['key'] == 'user') + cumulative_users_chart = next( + chart for chart in response.context['time_charts'] if chart['key'] == 'cumulative-user' + ) catalog_chart = next(chart for chart in response.context['category_charts'] if chart['key'] == 'catalog') progress_chart = next(chart for chart in response.context['category_charts'] if chart['key'] == 'project-progress') expected_date = timezone.localtime(project.created).date().isoformat() assert project_chart['total'] == 2 + assert project_chart['x_axis_title'] == 'Date of creation' + assert user_chart['x_axis_title'] == 'Date of registration' + assert cumulative_users_chart['x_axis_title'] == 'Date of registration' + assert response.context['summary'] == { + 'projects': 2, + 'users': 1, + 'catalogs_in_use': 1, + 'complete_projects': 0, + } assert project_chart['statistics']['day']['rows'] == [ {'key': expected_date, 'label': expected_date, 'value': 2}, ] assert user_chart['total'] == 1 assert set(catalog_chart['statistics']) == {'rows'} assert catalog_chart['chart_type'] == 'bar' - assert progress_chart['statistics']['rows'] == [ - {'key': 50, 'label': '50%', 'value': 2}, - ] - assert progress_chart['chart_type'] == 'scatter' + assert len(progress_chart['statistics']['rows']) == 11 + assert progress_chart['statistics']['rows'][5] == {'key': 50, 'label': '50-59%', 'value': 2} + assert progress_chart['chart_type'] == 'bar' assert progress_chart['orientation'] == 'vertical' assert progress_chart['x_axis_title'] == 'Progress (%)' assert progress_chart['y_axis_title'] == 'Number of projects' - assert b'data-chart-type="scatter"' in response.content + assert b'data-chart-type="bar"' in response.content assert b'data-chart-orientation="vertical"' in response.content + assert response.content.count(b'data-statistics-time-controls') == 1 + assert response.content.count(b'class="statistics-data-table"') == 5 + assert response.content.count(b'role="img"') == 5 + assert response.content.count(b'class="statistics-interval"') == 1 + assert b'