diff --git a/_doc/articles/2026-11-04-technique.rst b/_doc/articles/2026-11-04-technique.rst
new file mode 100644
index 0000000..361e680
--- /dev/null
+++ b/_doc/articles/2026-11-04-technique.rst
@@ -0,0 +1,166 @@
+2026-11-04 - une technique pour chaque échelle, chaque problème
+===============================================================
+
+**Plan**
+
+* Partie 1 : voyage technique d'un POC jusqu'à une grosse entreprise de données
+* Partie 2 : quelques cas pratiques
+
+**Cas pratiques**
+
+* Monter un petit dashboard avec github
+* Comment le faire sans gituhb ?
+* Spark et Map Reduce, une logique différente pour écrire des calculs
+
+**L'IA ou les agents**
+
+Qu'est ce qui change ?
+
+A-t-on encore besoin d'un outil comme Grafana ?
+
+- POC
+- Production ?
+- Dépendance ?
+
+**De la startup acoustique à la plateforme mondiale de données**
+
+- des données qu'on collecte
+- des données qu'on produit avec les outils collectant les données (pannes, ...)
+- lire et écrire en même temps
+- pandas / polars
+- ClickHouse / PostgreSQL
+
+30 étapes de développement technologique et industriel.
+
++----+------------------------------+--------------------------------------+------------------------------+
+| # | Évolution | Techniques, langages et technologies | Échelle |
++====+==============================+======================================+==============================+
+| 1 | POC acoustique : mesurer le | Python, NumPy, SciPy, PyAudio, FFT | 1 utilisateur, 1 capteur |
+| | bruit en continu chez un | | |
+| | particulier. | | |
++----+------------------------------+--------------------------------------+------------------------------+
+| 2 | Analyse du paysage sonore : | DSP, filtres numériques, | 1 environnement, plusieurs |
+| | identifier fréquences et | spectrogrammes, STFT | jours de mesures |
+| | événements. | | |
++----+------------------------------+--------------------------------------+------------------------------+
+| 3 | Premier capteur autonome : | Raspberry Pi, Linux, C++, | 1 à 10 équipements |
+| | collecte 24h/24 sur mini-PC. | systemd, ALSA | |
++----+------------------------------+--------------------------------------+------------------------------+
+| 4 | Stockage des mesures et | SQLite, PostgreSQL, SQL, | Millions de mesures |
+| | constitution d'un historique.| TimescaleDB | |
++----+------------------------------+--------------------------------------+------------------------------+
+| 5 | Première interface : | Python, Streamlit, Plotly, Grafana | Quelques utilisateurs |
+| | visualisation temporelle et | | |
+| | fréquentielle. | | |
++----+------------------------------+--------------------------------------+------------------------------+
+| 6 | Fiabilisation : gestion des | C++, Python, watchdog, buffers, | 10 à 100 capteurs |
+| | coupures, redémarrages et | retry, stockage local | |
+| | pertes de données. | | |
++----+------------------------------+--------------------------------------+------------------------------+
+| 7 | Réseau de capteurs : | MQTT, TLS, HTTP, certificats, | 100 à 1 000 capteurs |
+| | communication et identité | provisioning | |
+| | des équipements. | | |
++----+------------------------------+--------------------------------------+------------------------------+
+| 8 | Collecte centralisée : | Python, FastAPI, PostgreSQL, REST | 1 000 capteurs |
+| | réception et exposition | | |
+| | des mesures par API. | | |
++----+------------------------------+--------------------------------------+------------------------------+
+| 9 | Gestion du parc : | Java, Spring Boot, PostgreSQL, | Plusieurs milliers |
+| | état, localisation, | API, gestion de configuration | d'équipements |
+| | versions et maintenance. | | |
++----+------------------------------+--------------------------------------+------------------------------+
+| 10 | Premier produit commercial : | Grafana, React, TypeScript, | Plusieurs sites, dizaines |
+| | dashboards multi-sites et | WebSockets | de clients |
+| | suivi en direct. | | |
++----+------------------------------+--------------------------------------+------------------------------+
+| 11 | Architecture distribuée : | Microservices, Java, Python, Docker | 10 000 capteurs |
+| | séparation ingestion, | | |
+| | stockage et calcul. | | |
++----+------------------------------+--------------------------------------+------------------------------+
+| 12 | Ingestion massive : | Apache Kafka, partitionnement, | 10 000 à 50 000 capteurs |
+| | absorber les flux sans | consumer groups, backpressure | |
+| | perte. | | |
++----+------------------------------+--------------------------------------+------------------------------+
+| 13 | Traitement temps réel : | Apache Flink, Kafka Streams, | 50 000 à 100 000 capteurs |
+| | détection et agrégation | fenêtres temporelles | |
+| | des événements. | | |
++----+------------------------------+--------------------------------------+------------------------------+
+| 14 | Gestion des alertes : | CEP, moteurs de règles, | Milliers d'événements |
+| | seuils, anomalies, | Alertmanager, notifications | par seconde |
+| | confirmation et escalade. | | |
++----+------------------------------+--------------------------------------+------------------------------+
+| 15 | Infrastructure résiliente : | Kubernetes, réplication, failover, | 100 000 capteurs, |
+| | haute disponibilité et | autoscaling, Prometheus | disponibilité 24/7 |
+| | reprise après incident. | | |
++----+------------------------------+--------------------------------------+------------------------------+
+| 16 | Data Lake : conservation | S3, Parquet, compression, | Plusieurs pétaoctets |
+| | des mesures brutes et | partitionnement | |
+| | enrichies. | | |
++----+------------------------------+--------------------------------------+------------------------------+
+| 17 | Calcul distribué : | Apache Spark, PySpark, Arrow | Millions de capteurs, |
+| | analyse historique et | | milliards de mesures |
+| | calcul parallèle. | | |
++----+------------------------------+--------------------------------------+------------------------------+
+| 18 | Catalogue de données : | Apache Iceberg, Delta Lake, | Plusieurs pétaoctets, |
+| | métadonnées, évolution | schema evolution, time travel | centaines de datasets |
+| | des schémas et versions. | | |
++----+------------------------------+--------------------------------------+------------------------------+
+| 19 | Recherche analytique : | Trino, ClickHouse, Elasticsearch, | Milliards à milliers de |
+| | requêtes rapides sur les | SQL | milliards d'enregistrements |
+| | données historiques. | | |
++----+------------------------------+--------------------------------------+------------------------------+
+| 20 | Optimisation des | C++, SIMD, Arrow, memory mapping, | Plusieurs pétaoctets, |
+| | performances et des coûts. | cache distribué | requêtes parallèles |
++----+------------------------------+--------------------------------------+------------------------------+
+| 21 | API de recherche : | Java, Python, REST, GraphQL, | Centaines de clients et |
+| | accès aux mesures et | OAuth2 | partenaires |
+| | analyses pour les tiers. | | |
++----+------------------------------+--------------------------------------+------------------------------+
+| 22 | Environnement scientifique : | JupyterHub, Python, Polars, | Centaines de chercheurs, |
+| | notebooks et calcul sur | Pandas, Spark | milliers de notebooks |
+| | données historiques. | | |
++----+------------------------------+--------------------------------------+------------------------------+
+| 23 | Dashboards temps réel : | WebSockets, React, TypeScript, | Milliers d'utilisateurs |
+| | cartes, événements et | Grafana | simultanés |
+| | alertes interactives. | | |
++----+------------------------------+--------------------------------------+------------------------------+
+| 24 | Machine Learning : | PyTorch, scikit-learn, modèles | Millions de séries |
+| | classification, anomalies | distribués | temporelles |
+| | et reconnaissance. | | |
++----+------------------------------+--------------------------------------+------------------------------+
+| 25 | Gouvernance des données : | Apache Atlas, lineage, RBAC, | Plusieurs organisations, |
+| | traçabilité, anonymisation, | chiffrement, RGPD | gestion multi-tenant |
+| | droits et conservation. | | |
++----+------------------------------+--------------------------------------+------------------------------+
+| 26 | Déploiement mondial : | Kubernetes multi-clusters, | Plusieurs pays, |
+| | administration de réseaux | Terraform, cloud, edge computing | centaines de milliers |
+| | géographiquement répartis. | | de capteurs |
++----+------------------------------+--------------------------------------+------------------------------+
+| 27 | Plateforme multi-clients : | Multi-tenancy, OAuth2, RBAC, | Milliers de clients |
+| | isolation, quotas et | API Gateway | |
+| | facturation à l'usage. | | |
++----+------------------------------+--------------------------------------+------------------------------+
+| 28 | Marketplace de données : | Data Catalog, APIs, abonnements, | Marché international, |
+| | commercialisation de flux, | gouvernance | centaines de partenaires |
+| | indicateurs et historiques. | | |
++----+------------------------------+--------------------------------------+------------------------------+
+| 29 | Jumeaux numériques : | Digital Twins, graphes, SIG, | Régions, métropoles, |
+| | représentation dynamique | modèles physiques | territoires |
+| | des environnements. | | |
++----+------------------------------+--------------------------------------+------------------------------+
+| 30 | Plateforme mondiale : | Architecture événementielle, | Millions de capteurs, |
+| | intelligence environnementale| Big Data, IA distribuée, | plusieurs continents, |
+| | et prédictive. | haute disponibilité | exploitation 24/7 |
++----+------------------------------+--------------------------------------+------------------------------+
+
+
+Capacités transversales
+-----------------------
+
+- Corrélation spatio-temporelle des observations de capteurs.
+- Fusion de données multi-capteurs (bruit, température, fumée, vibrations,
+ humidité, pression, météo).
+- Détection et suivi de propagation d'incendies, d'inondations et de pollutions.
+- Gestion des alertes, confirmation multi-capteurs et escalade.
+- Recherche scientifique sur des milliards de mesures.
+- Résilience, réplication et reprise automatique après panne.
diff --git a/_doc/articles/index.rst b/_doc/articles/index.rst
index 84285a8..be08ee3 100644
--- a/_doc/articles/index.rst
+++ b/_doc/articles/index.rst
@@ -10,6 +10,7 @@ Ou *blog*.
:caption: 2026
:maxdepth: 1
+ 2026-11-04-technique
2026-04-30-route2026
.. toctree::
diff --git a/_doc/practice/spark/spark_matrix_3_columns.ipynb b/_doc/practice/spark/spark_matrix_3_columns.ipynb
index 0b9318f..0dbd5be 100644
--- a/_doc/practice/spark/spark_matrix_3_columns.ipynb
+++ b/_doc/practice/spark/spark_matrix_3_columns.ipynb
@@ -36,12 +36,14 @@
"text": [
"WARNING: Using incubator modules: jdk.incubator.vector\n",
"Using Spark's default log4j profile: org/apache/spark/log4j2-defaults.properties\n",
- "25/11/04 22:03:48 WARN Utils: Your hostname, xadupre2025, resolves to a loopback address: 127.0.1.1; using 172.17.197.78 instead (on interface eth0)\n",
- "25/11/04 22:03:48 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n",
+ "26/11/04 09:14:15 WARN Utils: Your hostname, xadupre2025, resolves to a loopback address: 127.0.1.1; using 172.28.185.142 instead (on interface eth0)\n",
+ "26/11/04 09:14:15 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n",
"Using Spark's default log4j profile: org/apache/spark/log4j2-defaults.properties\n",
"Setting default log level to \"WARN\".\n",
"To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n",
- "25/11/04 22:03:51 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n"
+ "/home/xadupre/vv/this312/lib/python3.12/site-packages/pyspark/testing/utils.py:127: FutureWarning: PySpark does not yet fully support pandas >= 3.0.0. Some features may not work correctly. It is recommended to use pandas < 3.0.0 for now.\n",
+ " require_minimum_pandas_version()\n",
+ "26/11/04 09:14:19 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n"
]
}
],
@@ -68,16 +70,16 @@
{
"data": {
"text/plain": [
- "array([[2.82769564, 4.3446237 ],\n",
- " [2.70903159, 4.28609675],\n",
- " [2.3197026 , 4.01486866],\n",
- " [2.61458882, 4.66117542],\n",
- " [1.99321382, 3.49690316],\n",
- " [3.08580871, 3.67840109],\n",
- " [2.99800173, 4.73972669],\n",
- " [3.55996446, 4.85961952],\n",
- " [2.51579451, 3.26530604],\n",
- " [2.21882233, 3.76041136]])"
+ "array([[2.29102946, 3.53847793],\n",
+ " [1.58850851, 2.78999386],\n",
+ " [1.40056768, 3.71886219],\n",
+ " [1.45873572, 3.49729928],\n",
+ " [2.2405258 , 3.47090722],\n",
+ " [1.15133072, 1.79002816],\n",
+ " [1.45400249, 3.65802814],\n",
+ " [2.55213675, 2.82495689],\n",
+ " [2.04015374, 2.78118401],\n",
+ " [2.64322208, 4.46851586]])"
]
},
"execution_count": 2,
@@ -126,53 +128,53 @@
"
\n",
" \n",
" | 0 | \n",
- " 0.452410 | \n",
- " 0.818571 | \n",
+ " 0.918575 | \n",
+ " 0.259207 | \n",
"
\n",
" \n",
" | 1 | \n",
- " 0.384679 | \n",
- " 0.965623 | \n",
+ " 0.123740 | \n",
+ " 0.395866 | \n",
"
\n",
" \n",
" | 2 | \n",
- " 0.646845 | \n",
- " 0.539604 | \n",
+ " 0.102176 | \n",
+ " 0.715428 | \n",
"
\n",
" \n",
" | 3 | \n",
- " 0.138227 | \n",
- " 0.917017 | \n",
+ " 0.573558 | \n",
+ " 0.249657 | \n",
"
\n",
" \n",
" | 4 | \n",
- " 0.422069 | \n",
- " 0.801644 | \n",
+ " 0.670691 | \n",
+ " 0.726082 | \n",
"
\n",
" \n",
" | 5 | \n",
- " 0.238986 | \n",
- " 0.595039 | \n",
+ " 0.039036 | \n",
+ " 0.889782 | \n",
"
\n",
" \n",
" | 6 | \n",
- " 0.895846 | \n",
- " 0.681612 | \n",
+ " 0.742978 | \n",
+ " 0.305118 | \n",
"
\n",
" \n",
" | 7 | \n",
- " 0.410572 | \n",
- " 0.862415 | \n",
+ " 0.193894 | \n",
+ " 0.969908 | \n",
"
\n",
" \n",
" | 8 | \n",
- " 0.869270 | \n",
- " 0.509816 | \n",
+ " 0.136791 | \n",
+ " 0.902031 | \n",
"
\n",
" \n",
" | 9 | \n",
- " 0.588944 | \n",
- " 0.955183 | \n",
+ " 0.229301 | \n",
+ " 0.530934 | \n",
"
\n",
" \n",
"\n",
@@ -180,16 +182,16 @@
],
"text/plain": [
" 0 1\n",
- "0 0.452410 0.818571\n",
- "1 0.384679 0.965623\n",
- "2 0.646845 0.539604\n",
- "3 0.138227 0.917017\n",
- "4 0.422069 0.801644\n",
- "5 0.238986 0.595039\n",
- "6 0.895846 0.681612\n",
- "7 0.410572 0.862415\n",
- "8 0.869270 0.509816\n",
- "9 0.588944 0.955183"
+ "0 0.918575 0.259207\n",
+ "1 0.123740 0.395866\n",
+ "2 0.102176 0.715428\n",
+ "3 0.573558 0.249657\n",
+ "4 0.670691 0.726082\n",
+ "5 0.039036 0.889782\n",
+ "6 0.742978 0.305118\n",
+ "7 0.193894 0.969908\n",
+ "8 0.136791 0.902031\n",
+ "9 0.229301 0.530934"
]
},
"execution_count": 3,
@@ -277,18 +279,18 @@
{
"data": {
"text/plain": [
- "[[0, 0, 0.03659454680020524],\n",
- " [0, 1, 0.767607811315502],\n",
- " [0, 2, 0.5733073849596189],\n",
- " [0, 3, 0.8757599092593937],\n",
- " [0, 4, 0.9367553854443735],\n",
- " [0, 5, 0.41209851669976916],\n",
- " [0, 6, 0.5540866293161243],\n",
- " [0, 7, 0.4665984547370805],\n",
- " [0, 8, 0.7567418978844924],\n",
- " [0, 9, 0.3129986409413059],\n",
- " [1, 0, 0.49628400193691236],\n",
- " [1, 1, 0.4944250595188363]]"
+ "[[0, 0, 0.4276345283939371],\n",
+ " [0, 1, 0.7104397235643978],\n",
+ " [0, 2, 0.5562289328230193],\n",
+ " [0, 3, 0.5985481810182711],\n",
+ " [0, 4, 0.7160591716006677],\n",
+ " [0, 5, 0.357351163573274],\n",
+ " [0, 6, 0.8440993774047496],\n",
+ " [0, 7, 0.9658516656607489],\n",
+ " [0, 8, 0.566678784552112],\n",
+ " [0, 9, 0.10479419449315441],\n",
+ " [1, 0, 0.3435094985776058],\n",
+ " [1, 1, 0.9361193497095325]]"
]
},
"execution_count": 8,
@@ -310,18 +312,18 @@
{
"data": {
"text/plain": [
- "[[0, 0, 0.452409743518958],\n",
- " [0, 1, 0.818571020089081],\n",
- " [1, 0, 0.3846793223675351],\n",
- " [1, 1, 0.9656232217356503],\n",
- " [2, 0, 0.6468448619803056],\n",
- " [2, 1, 0.5396040456507285],\n",
- " [3, 0, 0.13822741516437753],\n",
- " [3, 1, 0.9170173659401171],\n",
- " [4, 0, 0.4220691346862584],\n",
- " [4, 1, 0.8016438317882735],\n",
- " [5, 0, 0.23898597387372977],\n",
- " [5, 1, 0.5950391411819209]]"
+ "[[0, 0, 0.9185747759250762],\n",
+ " [0, 1, 0.2592066596075602],\n",
+ " [1, 0, 0.12374028350879862],\n",
+ " [1, 1, 0.39586600274279726],\n",
+ " [2, 0, 0.10217587658941296],\n",
+ " [2, 1, 0.7154281868182252],\n",
+ " [3, 0, 0.5735578686875115],\n",
+ " [3, 1, 0.24965710575045208],\n",
+ " [4, 0, 0.6706906195358108],\n",
+ " [4, 1, 0.7260817300612702],\n",
+ " [5, 0, 0.039036422428009976],\n",
+ " [5, 1, 0.8897817999507714]]"
]
},
"execution_count": 9,
@@ -359,18 +361,18 @@
{
"data": {
"text/plain": [
- "[(0, ((0, 0.03659454680020524), (0, 0.452409743518958))),\n",
- " (0, ((0, 0.03659454680020524), (1, 0.818571020089081))),\n",
- " (0, ((1, 0.49628400193691236), (0, 0.452409743518958))),\n",
- " (0, ((1, 0.49628400193691236), (1, 0.818571020089081))),\n",
- " (0, ((2, 0.26823832256619184), (0, 0.452409743518958))),\n",
- " (0, ((2, 0.26823832256619184), (1, 0.818571020089081))),\n",
- " (0, ((3, 0.8516746510081805), (0, 0.452409743518958))),\n",
- " (0, ((3, 0.8516746510081805), (1, 0.818571020089081))),\n",
- " (0, ((4, 0.210077377768215), (0, 0.452409743518958))),\n",
- " (0, ((4, 0.210077377768215), (1, 0.818571020089081))),\n",
- " (0, ((5, 0.0024981117896399896), (0, 0.452409743518958))),\n",
- " (0, ((5, 0.0024981117896399896), (1, 0.818571020089081)))]"
+ "[(0, ((0, 0.4276345283939371), (0, 0.9185747759250762))),\n",
+ " (0, ((0, 0.4276345283939371), (1, 0.2592066596075602))),\n",
+ " (0, ((1, 0.3435094985776058), (0, 0.9185747759250762))),\n",
+ " (0, ((1, 0.3435094985776058), (1, 0.2592066596075602))),\n",
+ " (0, ((2, 0.061827447722926254), (0, 0.9185747759250762))),\n",
+ " (0, ((2, 0.061827447722926254), (1, 0.2592066596075602))),\n",
+ " (0, ((3, 0.18643464032502244), (0, 0.9185747759250762))),\n",
+ " (0, ((3, 0.18643464032502244), (1, 0.2592066596075602))),\n",
+ " (0, ((4, 0.5724205231773295), (0, 0.9185747759250762))),\n",
+ " (0, ((4, 0.5724205231773295), (1, 0.2592066596075602))),\n",
+ " (0, ((5, 0.4975704022747822), (0, 0.9185747759250762))),\n",
+ " (0, ((5, 0.4975704022747822), (1, 0.2592066596075602)))]"
]
},
"execution_count": 10,
@@ -406,18 +408,18 @@
{
"data": {
"text/plain": [
- "[(0, 0, 0.01655572953207336),\n",
- " (0, 1, 0.029955235503941618),\n",
- " (1, 0, 0.22452371802884058),\n",
- " (1, 1, 0.4062437017193898),\n",
- " (2, 0, 0.12135363071412639),\n",
- " (2, 1, 0.21957211732999163),\n",
- " (3, 0, 0.38530591042420903),\n",
- " (3, 1, 0.6971561878597784),\n",
- " (4, 0, 0.0950410525952534),\n",
- " (4, 1, 0.17196325341736698),\n",
- " (5, 0, 0.001130170114032713),\n",
- " (5, 1, 0.002044881915942166)]"
+ "[(0, 0, 0.39281429109728644),\n",
+ " (0, 1, 0.11084571763784679),\n",
+ " (1, 0, 0.31553916068405957),\n",
+ " (1, 1, 0.08903994966976915),\n",
+ " (2, 0, 0.056793133938106345),\n",
+ " (2, 1, 0.016026086196320767),\n",
+ " (3, 0, 0.17125415796122967),\n",
+ " (3, 1, 0.048325100353786),\n",
+ " (4, 0, 0.5258110538125303),\n",
+ " (4, 1, 0.14837521170360757),\n",
+ " (5, 0, 0.4570556207765081),\n",
+ " (5, 1, 0.12897356189323625)]"
]
},
"execution_count": 11,
@@ -457,26 +459,26 @@
{
"data": {
"text/plain": [
- "[((0, 0), 2.8276956385710266),\n",
- " ((0, 1), 4.344623698461026),\n",
- " ((1, 0), 2.709031589561176),\n",
- " ((1, 1), 4.286096753861166),\n",
- " ((2, 0), 2.319702595813269),\n",
- " ((2, 1), 4.014868661312692),\n",
- " ((3, 0), 2.614588815326311),\n",
- " ((3, 1), 4.66117541697328),\n",
- " ((4, 0), 1.9932138244874387),\n",
- " ((4, 1), 3.4969031578212193),\n",
- " ((5, 0), 3.0858087055402157),\n",
- " ((5, 1), 3.678401085869831),\n",
- " ((6, 0), 2.9980017336305025),\n",
- " ((6, 1), 4.739726686558273),\n",
- " ((7, 0), 3.5599644578749863),\n",
- " ((7, 1), 4.859619515198687),\n",
- " ((8, 0), 2.515794511441488),\n",
- " ((8, 1), 3.26530604000568),\n",
- " ((9, 0), 2.218822328336375),\n",
- " ((9, 1), 3.7604113556777747)]"
+ "[((0, 0), 2.2910294615369833),\n",
+ " ((0, 1), 3.5384779259752115),\n",
+ " ((1, 0), 1.588508508773101),\n",
+ " ((1, 1), 2.7899938642491398),\n",
+ " ((2, 0), 1.400567677321856),\n",
+ " ((2, 1), 3.7188621947679996),\n",
+ " ((3, 0), 1.4587357182350686),\n",
+ " ((3, 1), 3.4972992791592126),\n",
+ " ((4, 0), 2.24052580041269),\n",
+ " ((4, 1), 3.470907222987571),\n",
+ " ((5, 0), 1.1513307163420712),\n",
+ " ((5, 1), 1.7900281634340787),\n",
+ " ((6, 0), 1.454002494006837),\n",
+ " ((6, 1), 3.658028135367462),\n",
+ " ((7, 0), 2.552136746927374),\n",
+ " ((7, 1), 2.8249568885507244),\n",
+ " ((8, 0), 2.0401537352713808),\n",
+ " ((8, 1), 2.7811840133848817),\n",
+ " ((9, 0), 2.6432220797579706),\n",
+ " ((9, 1), 4.468515860388137)]"
]
},
"execution_count": 12,
@@ -508,16 +510,16 @@
{
"data": {
"text/plain": [
- "array([[2.82769564, 4.3446237 ],\n",
- " [2.70903159, 4.28609675],\n",
- " [2.3197026 , 4.01486866],\n",
- " [2.61458882, 4.66117542],\n",
- " [1.99321382, 3.49690316],\n",
- " [3.08580871, 3.67840109],\n",
- " [2.99800173, 4.73972669],\n",
- " [3.55996446, 4.85961952],\n",
- " [2.51579451, 3.26530604],\n",
- " [2.21882233, 3.76041136]])"
+ "array([[2.29102946, 3.53847793],\n",
+ " [1.58850851, 2.78999386],\n",
+ " [1.40056768, 3.71886219],\n",
+ " [1.45873572, 3.49729928],\n",
+ " [2.2405258 , 3.47090722],\n",
+ " [1.15133072, 1.79002816],\n",
+ " [1.45400249, 3.65802814],\n",
+ " [2.55213675, 2.82495689],\n",
+ " [2.04015374, 2.78118401],\n",
+ " [2.64322208, 4.46851586]])"
]
},
"execution_count": 13,
@@ -544,7 +546,18 @@
"cell_type": "code",
"execution_count": 14,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/home/xadupre/vv/this312/lib/python3.12/site-packages/pyspark/sql/pandas/conversion.py:676: FutureWarning: PySpark does not yet fully support pandas >= 3.0.0. Some features may not work correctly. It is recommended to use pandas < 3.0.0 for now.\n",
+ " require_minimum_pandas_version()\n",
+ "/home/xadupre/vv/this312/lib/python3.12/site-packages/pyspark/sql/pandas/conversion.py:953: FutureWarning: PySpark does not yet fully support pandas >= 3.0.0. Some features may not work correctly. It is recommended to use pandas < 3.0.0 for now.\n",
+ " require_minimum_pandas_version()\n"
+ ]
+ }
+ ],
"source": [
"schema = [\"index\"] + [\"c%d\" % i for i in range(1, 11)]\n",
"mat1 = spark.createDataFrame(\n",
@@ -697,6 +710,22 @@
"execution_count": 21,
"metadata": {},
"outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/home/xadupre/vv/this312/lib/python3.12/site-packages/pyspark/sql/pandas/conversion.py:324: FutureWarning: PySpark does not yet fully support pandas >= 3.0.0. Some features may not work correctly. It is recommended to use pandas < 3.0.0 for now.\n",
+ " require_minimum_pandas_version()\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/home/xadupre/vv/this312/lib/python3.12/site-packages/pyspark/sql/pandas/conversion.py:324: FutureWarning: PySpark does not yet fully support pandas >= 3.0.0. Some features may not work correctly. It is recommended to use pandas < 3.0.0 for now.\n",
+ " require_minimum_pandas_version()\n"
+ ]
+ },
{
"data": {
"text/plain": [
@@ -786,7 +815,8 @@
"name": "stderr",
"output_type": "stream",
"text": [
- " \r"
+ "/home/xadupre/vv/this312/lib/python3.12/site-packages/pyspark/sql/pandas/conversion.py:324: FutureWarning: PySpark does not yet fully support pandas >= 3.0.0. Some features may not work correctly. It is recommended to use pandas < 3.0.0 for now.\n",
+ " require_minimum_pandas_version()\n"
]
},
{
@@ -823,46 +853,46 @@
" 0 | \n",
" 0 | \n",
" 0 | \n",
- " 0.036595 | \n",
+ " 0.427635 | \n",
" 0 | \n",
- " 0 | \n",
- " 0.452410 | \n",
+ " 1 | \n",
+ " 0.259207 | \n",
" \n",
" \n",
" | 1 | \n",
" 0 | \n",
" 0 | \n",
- " 0.036595 | \n",
+ " 0.427635 | \n",
" 0 | \n",
- " 1 | \n",
- " 0.818571 | \n",
+ " 0 | \n",
+ " 0.918575 | \n",
"
\n",
" \n",
" | 2 | \n",
- " 1 | \n",
- " 0 | \n",
- " 0.496284 | \n",
" 0 | \n",
- " 0 | \n",
- " 0.452410 | \n",
+ " 1 | \n",
+ " 0.710440 | \n",
+ " 1 | \n",
+ " 1 | \n",
+ " 0.395866 | \n",
"
\n",
" \n",
" | 3 | \n",
- " 1 | \n",
- " 0 | \n",
- " 0.496284 | \n",
" 0 | \n",
" 1 | \n",
- " 0.818571 | \n",
+ " 0.710440 | \n",
+ " 1 | \n",
+ " 0 | \n",
+ " 0.123740 | \n",
"
\n",
" \n",
" | 4 | \n",
- " 2 | \n",
- " 0 | \n",
- " 0.268238 | \n",
" 0 | \n",
- " 0 | \n",
- " 0.452410 | \n",
+ " 2 | \n",
+ " 0.556229 | \n",
+ " 2 | \n",
+ " 1 | \n",
+ " 0.715428 | \n",
"
\n",
" \n",
"\n",
@@ -870,11 +900,11 @@
],
"text/plain": [
" index pos col index2 pos2 col2\n",
- "0 0 0 0.036595 0 0 0.452410\n",
- "1 0 0 0.036595 0 1 0.818571\n",
- "2 1 0 0.496284 0 0 0.452410\n",
- "3 1 0 0.496284 0 1 0.818571\n",
- "4 2 0 0.268238 0 0 0.452410"
+ "0 0 0 0.427635 0 1 0.259207\n",
+ "1 0 0 0.427635 0 0 0.918575\n",
+ "2 0 1 0.710440 1 1 0.395866\n",
+ "3 0 1 0.710440 1 0 0.123740\n",
+ "4 0 2 0.556229 2 1 0.715428"
]
},
"execution_count": 25,
@@ -925,7 +955,16 @@
"cell_type": "code",
"execution_count": 28,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/home/xadupre/vv/this312/lib/python3.12/site-packages/pyspark/sql/pandas/conversion.py:324: FutureWarning: PySpark does not yet fully support pandas >= 3.0.0. Some features may not work correctly. It is recommended to use pandas < 3.0.0 for now.\n",
+ " require_minimum_pandas_version()\n"
+ ]
+ }
+ ],
"source": [
"df = final.toPandas()"
]
@@ -963,46 +1002,46 @@
" \n",
" \n",
" \n",
- " | 7 | \n",
+ " 0 | \n",
" 0 | \n",
" 0 | \n",
- " 2.827696 | \n",
+ " 2.291029 | \n",
"
\n",
" \n",
- " | 10 | \n",
+ " 1 | \n",
" 0 | \n",
" 1 | \n",
- " 4.344624 | \n",
+ " 3.538478 | \n",
"
\n",
" \n",
- " | 18 | \n",
+ " 3 | \n",
" 1 | \n",
" 0 | \n",
- " 2.709032 | \n",
+ " 1.588509 | \n",
"
\n",
" \n",
- " | 3 | \n",
+ " 2 | \n",
" 1 | \n",
" 1 | \n",
- " 4.286097 | \n",
+ " 2.789994 | \n",
"
\n",
" \n",
- " | 6 | \n",
+ " 4 | \n",
" 2 | \n",
" 0 | \n",
- " 2.319703 | \n",
+ " 1.400568 | \n",
"
\n",
" \n",
"\n",
""
],
"text/plain": [
- " i j sum(val)\n",
- "7 0 0 2.827696\n",
- "10 0 1 4.344624\n",
- "18 1 0 2.709032\n",
- "3 1 1 4.286097\n",
- "6 2 0 2.319703"
+ " i j sum(val)\n",
+ "0 0 0 2.291029\n",
+ "1 0 1 3.538478\n",
+ "3 1 0 1.588509\n",
+ "2 1 1 2.789994\n",
+ "4 2 0 1.400568"
]
},
"execution_count": 29,
@@ -1060,11 +1099,16 @@
"source": [
"spark.stop()"
]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": []
}
],
"metadata": {
"kernelspec": {
- "display_name": "this312",
+ "display_name": "this312 (3.12.3)",
"language": "python",
"name": "python3"
},
diff --git a/_unittests/ut_xrun_doc/test_setup.py b/_unittests/ut_xrun_doc/test_setup.py
new file mode 100644
index 0000000..0e31138
--- /dev/null
+++ b/_unittests/ut_xrun_doc/test_setup.py
@@ -0,0 +1,48 @@
+import subprocess
+import sys
+import unittest
+from pathlib import Path
+
+
+class TestSetup(unittest.TestCase):
+ def test_torch_paths_without_cuda_probe(self):
+ setup = Path(__file__).resolve().parents[2] / "setup.py"
+ script = """
+import sys
+from pathlib import Path
+import setuptools
+from distutils.core import run_setup
+import torch
+
+
+def unexpected_cuda_probe():
+ raise AssertionError("Finding build paths must not probe the CUDA driver")
+
+
+torch.cuda.is_available = unexpected_cuda_probe
+distribution = run_setup(sys.argv[1], stop_after="config")
+command = distribution.get_command_obj("build_ext")
+command.use_cuda = False
+command.ensure_finalized()
+definitions = dict(
+ argument[2:].split("=", 1)
+ for argument in command.get_cmake_args("Release")
+ if argument.startswith("-D")
+)
+includes = [Path(path) for path in definitions["TORCH_INCLUDE"].split(";")]
+for header in ["torch/extension.h", "torch/torch.h"]:
+ assert any((path / header).is_file() for path in includes), header
+assert Path(definitions["TORCH_LIBRARIES_DIR"]).is_dir()
+assert Path(definitions["CMAKE_PREFIX_PATH"]).is_dir()
+"""
+ result = subprocess.run(
+ [sys.executable, "-c", script, str(setup)],
+ cwd=setup.parent,
+ capture_output=True,
+ text=True,
+ )
+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+
+
+if __name__ == "__main__":
+ unittest.main(verbosity=2)
diff --git a/pyproject.toml b/pyproject.toml
index 3923e8b..b626cb3 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -129,7 +129,7 @@ test-command = "python -c \"import teachcompute;teachcompute.check_installation(
[tool.cibuildwheel.linux]
archs = ["x86_64"]
build = "cp*"
-skip = "cp36-* cp37-* cp38-* cp39-* cp314-* cp314t-* cp315-* pypy* *musllinux*"
+skip = "cp36-* cp37-* cp38-* cp39-* cp314-* cp314t-* cp315-* cp315t-* pypy* *musllinux*"
manylinux-x86_64-image = "manylinux_2_28"
before-build = "pip install auditwheel-symbols abi3audit;python -c 'import sysconfig;print(sysconfig.get_platform())'"
build-verbosity = 1
@@ -148,13 +148,17 @@ environment = """
"""
build = "cp*"
before-build = "brew install llvm libomp"
-skip = "pypy* pp* cp36-* cp37-* cp38-* cp39-* cp310-* cp314-* cp314t-* cp315-*"
+skip = "pypy* pp* cp36-* cp37-* cp38-* cp39-* cp310-* cp314-* cp314t-* cp315-* cp315t-*"
# environment="LDFLAGS='-L/opt/homebrew/opt/llvm/lib -L/opt/homebrew/opt/libomp/lib' CPPFLAGS='-I/opt/homebrew/opt/llvm/include -I/opt/homebrew/opt/libomp/include'"
[tool.cibuildwheel.windows]
archs = ["AMD64"]
build = "cp*"
-skip = "cp36-* cp37-* cp38-* cp39-* cp314-* cp314t-* cp315-* pypy*"
+skip = "cp36-* cp37-* cp38-* cp39-* cp314-* cp314t-* cp315-* cp315t-* pypy*"
+# PyTorch loads its own DLLs before importing the extension.
+repair-wheel-command = "delvewheel repair -w {dest_dir} -v --exclude c10.dll;torch.dll;torch_cpu.dll;torch_python.dll {wheel}"
+test-requires = ["torch"]
+test-command = "python -c \"import teachcompute;teachcompute.check_installation(val=True)\" && python {project}/_unittests/ut_torch_extensions/test_fcts_piecewise_linear.py"
[tool.cython-lint]
max-line-length = 88
diff --git a/requirements-dev.txt b/requirements-dev.txt
index 5e2397d..de79110 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -1,3 +1,6 @@
+--find-links https://github.com/xadupre/onnx-light/releases/expanded_assets/0.2.6
+# Select the full wheel rather than the reduced build on Linux CI.
+onnx-light @ https://github.com/xadupre/onnx-light/releases/download/0.2.6/onnx_light-0.2.6-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl ; python_version == "3.12" and sys_platform == "linux" and platform_machine == "x86_64"
black
black-nb
clang-format
diff --git a/setup.py b/setup.py
index 5079ac9..b9df50f 100644
--- a/setup.py
+++ b/setup.py
@@ -215,15 +215,19 @@ class cmake_build_class_extension(Command):
(
"use-cuda=",
None,
- "If cuda is available, CUDA is "
- "used by default unless this option is set to 0",
+ (
+ "If cuda is available, CUDA is "
+ "used by default unless this option is set to 0"
+ ),
),
("use-nvtx=", None, "Enables compilation with NVTX events."),
(
"cuda-version=",
None,
- "If cuda is available, it searches the installed version "
- "unless this option is defined.",
+ (
+ "If cuda is available, it searches the installed version "
+ "unless this option is defined."
+ ),
),
(
"parallel=",
@@ -233,17 +237,21 @@ class cmake_build_class_extension(Command):
(
"cuda-build=",
None,
- "CUDA code can be compiled to be working with "
- "different architectures, this flag can optimize "
- "for a specific machine, possible values: DEFAULT, "
- "H100, H100opt",
+ (
+ "CUDA code can be compiled to be working with "
+ "different architectures, this flag can optimize "
+ "for a specific machine, possible values: DEFAULT, "
+ "H100, H100opt"
+ ),
),
(
"cuda-link=",
None,
- "CUDA can statically linked (STATIC) or dynamically "
- "(SHARED), default is STATIC."
- "STATIC",
+ (
+ "CUDA can statically linked (STATIC) or dynamically "
+ "(SHARED), default is STATIC."
+ "STATIC"
+ ),
),
(
"cuda-nvcc=",
@@ -411,10 +419,18 @@ def get_cmake_args(self, cfg: str) -> List[str]:
try:
import torch.utils
- from torch.utils.cpp_extension import include_paths, library_paths
- torch_include = ";".join(include_paths())
- torch_libraries = ";".join(library_paths())
+ # cpp_extension probes the CUDA driver even for CPU header paths.
+ torch_dir = os.path.dirname(torch.__file__)
+ torch_include = ";".join(
+ [
+ os.path.join(torch_dir, "include"),
+ os.path.join(
+ torch_dir, "include", "torch", "csrc", "api", "include"
+ ),
+ ]
+ )
+ torch_libraries = os.path.join(torch_dir, "lib")
torch_prefix = torch.utils.cmake_prefix_path
except ImportError:
torch_include = None