From 039d60997b8faa7640bae37653c4896cd4cc6849 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= Date: Wed, 4 Nov 2026 09:12:27 +0100 Subject: [PATCH 1/6] fix --- _doc/articles/2026-11-04-technique.rst | 140 +++++++++++++++++++++++++ _doc/articles/index.rst | 1 + 2 files changed, 141 insertions(+) create mode 100644 _doc/articles/2026-11-04-technique.rst diff --git a/_doc/articles/2026-11-04-technique.rst b/_doc/articles/2026-11-04-technique.rst new file mode 100644 index 0000000..c3980ca --- /dev/null +++ b/_doc/articles/2026-11-04-technique.rst @@ -0,0 +1,140 @@ +2026-11-04 - une technique pour chaque échelle, chaque problème + + +**De la startup acoustique à la plateforme mondiale de données** + + +30 étapes de développement technologique et industriel. + ++----+------------------------------+--------------------------------------+------------------------------+ +| # | Évolution | Techniques, langages et technologies | Échelle | ++====+==============================+======================================+==============================+ +| 1 | POC acoustique : mesurer le | Python, NumPy, SciPy, PyAudio, FFT | 1 utilisateur, 1 capteur | +| | bruit en continu chez un | | | +| | particulier. | | | ++----+------------------------------+--------------------------------------+------------------------------+ +| 2 | Analyse du paysage sonore : | DSP, filtres numériques, | 1 environnement, plusieurs | +| | identifier fréquences et | spectrogrammes, STFT | jours de mesures | +| | événements. | | | ++----+------------------------------+--------------------------------------+------------------------------+ +| 3 | Premier capteur autonome : | Raspberry Pi, Linux, C++, | 1 à 10 équipements | +| | collecte 24h/24 sur mini-PC. | systemd, ALSA | | ++----+------------------------------+--------------------------------------+------------------------------+ +| 4 | Stockage des mesures et | SQLite, PostgreSQL, SQL, | Millions de mesures | +| | constitution d'un historique.| TimescaleDB | | ++----+------------------------------+--------------------------------------+------------------------------+ +| 5 | Première interface : | Python, Streamlit, Plotly, Grafana | Quelques utilisateurs | +| | visualisation temporelle et | | | +| | fréquentielle. | | | ++----+------------------------------+--------------------------------------+------------------------------+ +| 6 | Fiabilisation : gestion des | C++, Python, watchdog, buffers, | 10 à 100 capteurs | +| | coupures, redémarrages et | retry, stockage local | | +| | pertes de données. | | | ++----+------------------------------+--------------------------------------+------------------------------+ +| 7 | Réseau de capteurs : | MQTT, TLS, HTTP, certificats, | 100 à 1 000 capteurs | +| | communication et identité | provisioning | | +| | des équipements. | | | ++----+------------------------------+--------------------------------------+------------------------------+ +| 8 | Collecte centralisée : | Python, FastAPI, PostgreSQL, REST | 1 000 capteurs | +| | réception et exposition | | | +| | des mesures par API. | | | ++----+------------------------------+--------------------------------------+------------------------------+ +| 9 | Gestion du parc : | Java, Spring Boot, PostgreSQL, | Plusieurs milliers | +| | état, localisation, | API, gestion de configuration | d'équipements | +| | versions et maintenance. | | | ++----+------------------------------+--------------------------------------+------------------------------+ +| 10 | Premier produit commercial : | Grafana, React, TypeScript, | Plusieurs sites, dizaines | +| | dashboards multi-sites et | WebSockets | de clients | +| | suivi en direct. | | | ++----+------------------------------+--------------------------------------+------------------------------+ +| 11 | Architecture distribuée : | Microservices, Java, Python, Docker | 10 000 capteurs | +| | séparation ingestion, | | | +| | stockage et calcul. | | | ++----+------------------------------+--------------------------------------+------------------------------+ +| 12 | Ingestion massive : | Apache Kafka, partitionnement, | 10 000 à 50 000 capteurs | +| | absorber les flux sans | consumer groups, backpressure | | +| | perte. | | | ++----+------------------------------+--------------------------------------+------------------------------+ +| 13 | Traitement temps réel : | Apache Flink, Kafka Streams, | 50 000 à 100 000 capteurs | +| | détection et agrégation | fenêtres temporelles | | +| | des événements. | | | ++----+------------------------------+--------------------------------------+------------------------------+ +| 14 | Gestion des alertes : | CEP, moteurs de règles, | Milliers d'événements | +| | seuils, anomalies, | Alertmanager, notifications | par seconde | +| | confirmation et escalade. | | | ++----+------------------------------+--------------------------------------+------------------------------+ +| 15 | Infrastructure résiliente : | Kubernetes, réplication, failover, | 100 000 capteurs, | +| | haute disponibilité et | autoscaling, Prometheus | disponibilité 24/7 | +| | reprise après incident. | | | ++----+------------------------------+--------------------------------------+------------------------------+ +| 16 | Data Lake : conservation | S3, Parquet, compression, | Plusieurs pétaoctets | +| | des mesures brutes et | partitionnement | | +| | enrichies. | | | ++----+------------------------------+--------------------------------------+------------------------------+ +| 17 | Calcul distribué : | Apache Spark, PySpark, Arrow | Millions de capteurs, | +| | analyse historique et | | milliards de mesures | +| | calcul parallèle. | | | ++----+------------------------------+--------------------------------------+------------------------------+ +| 18 | Catalogue de données : | Apache Iceberg, Delta Lake, | Plusieurs pétaoctets, | +| | métadonnées, évolution | schema evolution, time travel | centaines de datasets | +| | des schémas et versions. | | | ++----+------------------------------+--------------------------------------+------------------------------+ +| 19 | Recherche analytique : | Trino, ClickHouse, Elasticsearch, | Milliards à milliers de | +| | requêtes rapides sur les | SQL | milliards d'enregistrements | +| | données historiques. | | | ++----+------------------------------+--------------------------------------+------------------------------+ +| 20 | Optimisation des | C++, SIMD, Arrow, memory mapping, | Plusieurs pétaoctets, | +| | performances et des coûts. | cache distribué | requêtes parallèles | ++----+------------------------------+--------------------------------------+------------------------------+ +| 21 | API de recherche : | Java, Python, REST, GraphQL, | Centaines de clients et | +| | accès aux mesures et | OAuth2 | partenaires | +| | analyses pour les tiers. | | | ++----+------------------------------+--------------------------------------+------------------------------+ +| 22 | Environnement scientifique : | JupyterHub, Python, Polars, | Centaines de chercheurs, | +| | notebooks et calcul sur | Pandas, Spark | milliers de notebooks | +| | données historiques. | | | ++----+------------------------------+--------------------------------------+------------------------------+ +| 23 | Dashboards temps réel : | WebSockets, React, TypeScript, | Milliers d'utilisateurs | +| | cartes, événements et | Grafana | simultanés | +| | alertes interactives. | | | ++----+------------------------------+--------------------------------------+------------------------------+ +| 24 | Machine Learning : | PyTorch, scikit-learn, modèles | Millions de séries | +| | classification, anomalies | distribués | temporelles | +| | et reconnaissance. | | | ++----+------------------------------+--------------------------------------+------------------------------+ +| 25 | Gouvernance des données : | Apache Atlas, lineage, RBAC, | Plusieurs organisations, | +| | traçabilité, anonymisation, | chiffrement, RGPD | gestion multi-tenant | +| | droits et conservation. | | | ++----+------------------------------+--------------------------------------+------------------------------+ +| 26 | Déploiement mondial : | Kubernetes multi-clusters, | Plusieurs pays, | +| | administration de réseaux | Terraform, cloud, edge computing | centaines de milliers | +| | géographiquement répartis. | | de capteurs | ++----+------------------------------+--------------------------------------+------------------------------+ +| 27 | Plateforme multi-clients : | Multi-tenancy, OAuth2, RBAC, | Milliers de clients | +| | isolation, quotas et | API Gateway | | +| | facturation à l'usage. | | | ++----+------------------------------+--------------------------------------+------------------------------+ +| 28 | Marketplace de données : | Data Catalog, APIs, abonnements, | Marché international, | +| | commercialisation de flux, | gouvernance | centaines de partenaires | +| | indicateurs et historiques. | | | ++----+------------------------------+--------------------------------------+------------------------------+ +| 29 | Jumeaux numériques : | Digital Twins, graphes, SIG, | Régions, métropoles, | +| | représentation dynamique | modèles physiques | territoires | +| | des environnements. | | | ++----+------------------------------+--------------------------------------+------------------------------+ +| 30 | Plateforme mondiale : | Architecture événementielle, | Millions de capteurs, | +| | intelligence environnementale| Big Data, IA distribuée, | plusieurs continents, | +| | et prédictive. | haute disponibilité | exploitation 24/7 | ++----+------------------------------+--------------------------------------+------------------------------+ + + +Capacités transversales +======================= + +- Corrélation spatio-temporelle des observations de capteurs. +- Fusion de données multi-capteurs (bruit, température, fumée, vibrations, + humidité, pression, météo). +- Détection et suivi de propagation d'incendies, d'inondations et de pollutions. +- Gestion des alertes, confirmation multi-capteurs et escalade. +- Recherche scientifique sur des milliards de mesures. +- Résilience, réplication et reprise automatique après panne. diff --git a/_doc/articles/index.rst b/_doc/articles/index.rst index 84285a8..be08ee3 100644 --- a/_doc/articles/index.rst +++ b/_doc/articles/index.rst @@ -10,6 +10,7 @@ Ou *blog*. :caption: 2026 :maxdepth: 1 + 2026-11-04-technique 2026-04-30-route2026 .. toctree:: From f108bab99b494b8522cf7bfb263fca5b3bb201fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= Date: Wed, 4 Nov 2026 09:15:07 +0100 Subject: [PATCH 2/6] update --- .../spark/spark_matrix_3_columns.ipynb | 376 ++++++++++-------- 1 file changed, 210 insertions(+), 166 deletions(-) diff --git a/_doc/practice/spark/spark_matrix_3_columns.ipynb b/_doc/practice/spark/spark_matrix_3_columns.ipynb index 0b9318f..0dbd5be 100644 --- a/_doc/practice/spark/spark_matrix_3_columns.ipynb +++ b/_doc/practice/spark/spark_matrix_3_columns.ipynb @@ -36,12 +36,14 @@ "text": [ "WARNING: Using incubator modules: jdk.incubator.vector\n", "Using Spark's default log4j profile: org/apache/spark/log4j2-defaults.properties\n", - "25/11/04 22:03:48 WARN Utils: Your hostname, xadupre2025, resolves to a loopback address: 127.0.1.1; using 172.17.197.78 instead (on interface eth0)\n", - "25/11/04 22:03:48 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n", + "26/11/04 09:14:15 WARN Utils: Your hostname, xadupre2025, resolves to a loopback address: 127.0.1.1; using 172.28.185.142 instead (on interface eth0)\n", + "26/11/04 09:14:15 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n", "Using Spark's default log4j profile: org/apache/spark/log4j2-defaults.properties\n", "Setting default log level to \"WARN\".\n", "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n", - "25/11/04 22:03:51 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n" + "/home/xadupre/vv/this312/lib/python3.12/site-packages/pyspark/testing/utils.py:127: FutureWarning: PySpark does not yet fully support pandas >= 3.0.0. Some features may not work correctly. It is recommended to use pandas < 3.0.0 for now.\n", + " require_minimum_pandas_version()\n", + "26/11/04 09:14:19 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n" ] } ], @@ -68,16 +70,16 @@ { "data": { "text/plain": [ - "array([[2.82769564, 4.3446237 ],\n", - " [2.70903159, 4.28609675],\n", - " [2.3197026 , 4.01486866],\n", - " [2.61458882, 4.66117542],\n", - " [1.99321382, 3.49690316],\n", - " [3.08580871, 3.67840109],\n", - " [2.99800173, 4.73972669],\n", - " [3.55996446, 4.85961952],\n", - " [2.51579451, 3.26530604],\n", - " [2.21882233, 3.76041136]])" + "array([[2.29102946, 3.53847793],\n", + " [1.58850851, 2.78999386],\n", + " [1.40056768, 3.71886219],\n", + " [1.45873572, 3.49729928],\n", + " [2.2405258 , 3.47090722],\n", + " [1.15133072, 1.79002816],\n", + " [1.45400249, 3.65802814],\n", + " [2.55213675, 2.82495689],\n", + " [2.04015374, 2.78118401],\n", + " [2.64322208, 4.46851586]])" ] }, "execution_count": 2, @@ -126,53 +128,53 @@ " \n", " \n", " 0\n", - " 0.452410\n", - " 0.818571\n", + " 0.918575\n", + " 0.259207\n", " \n", " \n", " 1\n", - " 0.384679\n", - " 0.965623\n", + " 0.123740\n", + " 0.395866\n", " \n", " \n", " 2\n", - " 0.646845\n", - " 0.539604\n", + " 0.102176\n", + " 0.715428\n", " \n", " \n", " 3\n", - " 0.138227\n", - " 0.917017\n", + " 0.573558\n", + " 0.249657\n", " \n", " \n", " 4\n", - " 0.422069\n", - " 0.801644\n", + " 0.670691\n", + " 0.726082\n", " \n", " \n", " 5\n", - " 0.238986\n", - " 0.595039\n", + " 0.039036\n", + " 0.889782\n", " \n", " \n", " 6\n", - " 0.895846\n", - " 0.681612\n", + " 0.742978\n", + " 0.305118\n", " \n", " \n", " 7\n", - " 0.410572\n", - " 0.862415\n", + " 0.193894\n", + " 0.969908\n", " \n", " \n", " 8\n", - " 0.869270\n", - " 0.509816\n", + " 0.136791\n", + " 0.902031\n", " \n", " \n", " 9\n", - " 0.588944\n", - " 0.955183\n", + " 0.229301\n", + " 0.530934\n", " \n", " \n", "\n", @@ -180,16 +182,16 @@ ], "text/plain": [ " 0 1\n", - "0 0.452410 0.818571\n", - "1 0.384679 0.965623\n", - "2 0.646845 0.539604\n", - "3 0.138227 0.917017\n", - "4 0.422069 0.801644\n", - "5 0.238986 0.595039\n", - "6 0.895846 0.681612\n", - "7 0.410572 0.862415\n", - "8 0.869270 0.509816\n", - "9 0.588944 0.955183" + "0 0.918575 0.259207\n", + "1 0.123740 0.395866\n", + "2 0.102176 0.715428\n", + "3 0.573558 0.249657\n", + "4 0.670691 0.726082\n", + "5 0.039036 0.889782\n", + "6 0.742978 0.305118\n", + "7 0.193894 0.969908\n", + "8 0.136791 0.902031\n", + "9 0.229301 0.530934" ] }, "execution_count": 3, @@ -277,18 +279,18 @@ { "data": { "text/plain": [ - "[[0, 0, 0.03659454680020524],\n", - " [0, 1, 0.767607811315502],\n", - " [0, 2, 0.5733073849596189],\n", - " [0, 3, 0.8757599092593937],\n", - " [0, 4, 0.9367553854443735],\n", - " [0, 5, 0.41209851669976916],\n", - " [0, 6, 0.5540866293161243],\n", - " [0, 7, 0.4665984547370805],\n", - " [0, 8, 0.7567418978844924],\n", - " [0, 9, 0.3129986409413059],\n", - " [1, 0, 0.49628400193691236],\n", - " [1, 1, 0.4944250595188363]]" + "[[0, 0, 0.4276345283939371],\n", + " [0, 1, 0.7104397235643978],\n", + " [0, 2, 0.5562289328230193],\n", + " [0, 3, 0.5985481810182711],\n", + " [0, 4, 0.7160591716006677],\n", + " [0, 5, 0.357351163573274],\n", + " [0, 6, 0.8440993774047496],\n", + " [0, 7, 0.9658516656607489],\n", + " [0, 8, 0.566678784552112],\n", + " [0, 9, 0.10479419449315441],\n", + " [1, 0, 0.3435094985776058],\n", + " [1, 1, 0.9361193497095325]]" ] }, "execution_count": 8, @@ -310,18 +312,18 @@ { "data": { "text/plain": [ - "[[0, 0, 0.452409743518958],\n", - " [0, 1, 0.818571020089081],\n", - " [1, 0, 0.3846793223675351],\n", - " [1, 1, 0.9656232217356503],\n", - " [2, 0, 0.6468448619803056],\n", - " [2, 1, 0.5396040456507285],\n", - " [3, 0, 0.13822741516437753],\n", - " [3, 1, 0.9170173659401171],\n", - " [4, 0, 0.4220691346862584],\n", - " [4, 1, 0.8016438317882735],\n", - " [5, 0, 0.23898597387372977],\n", - " [5, 1, 0.5950391411819209]]" + "[[0, 0, 0.9185747759250762],\n", + " [0, 1, 0.2592066596075602],\n", + " [1, 0, 0.12374028350879862],\n", + " [1, 1, 0.39586600274279726],\n", + " [2, 0, 0.10217587658941296],\n", + " [2, 1, 0.7154281868182252],\n", + " [3, 0, 0.5735578686875115],\n", + " [3, 1, 0.24965710575045208],\n", + " [4, 0, 0.6706906195358108],\n", + " [4, 1, 0.7260817300612702],\n", + " [5, 0, 0.039036422428009976],\n", + " [5, 1, 0.8897817999507714]]" ] }, "execution_count": 9, @@ -359,18 +361,18 @@ { "data": { "text/plain": [ - "[(0, ((0, 0.03659454680020524), (0, 0.452409743518958))),\n", - " (0, ((0, 0.03659454680020524), (1, 0.818571020089081))),\n", - " (0, ((1, 0.49628400193691236), (0, 0.452409743518958))),\n", - " (0, ((1, 0.49628400193691236), (1, 0.818571020089081))),\n", - " (0, ((2, 0.26823832256619184), (0, 0.452409743518958))),\n", - " (0, ((2, 0.26823832256619184), (1, 0.818571020089081))),\n", - " (0, ((3, 0.8516746510081805), (0, 0.452409743518958))),\n", - " (0, ((3, 0.8516746510081805), (1, 0.818571020089081))),\n", - " (0, ((4, 0.210077377768215), (0, 0.452409743518958))),\n", - " (0, ((4, 0.210077377768215), (1, 0.818571020089081))),\n", - " (0, ((5, 0.0024981117896399896), (0, 0.452409743518958))),\n", - " (0, ((5, 0.0024981117896399896), (1, 0.818571020089081)))]" + "[(0, ((0, 0.4276345283939371), (0, 0.9185747759250762))),\n", + " (0, ((0, 0.4276345283939371), (1, 0.2592066596075602))),\n", + " (0, ((1, 0.3435094985776058), (0, 0.9185747759250762))),\n", + " (0, ((1, 0.3435094985776058), (1, 0.2592066596075602))),\n", + " (0, ((2, 0.061827447722926254), (0, 0.9185747759250762))),\n", + " (0, ((2, 0.061827447722926254), (1, 0.2592066596075602))),\n", + " (0, ((3, 0.18643464032502244), (0, 0.9185747759250762))),\n", + " (0, ((3, 0.18643464032502244), (1, 0.2592066596075602))),\n", + " (0, ((4, 0.5724205231773295), (0, 0.9185747759250762))),\n", + " (0, ((4, 0.5724205231773295), (1, 0.2592066596075602))),\n", + " (0, ((5, 0.4975704022747822), (0, 0.9185747759250762))),\n", + " (0, ((5, 0.4975704022747822), (1, 0.2592066596075602)))]" ] }, "execution_count": 10, @@ -406,18 +408,18 @@ { "data": { "text/plain": [ - "[(0, 0, 0.01655572953207336),\n", - " (0, 1, 0.029955235503941618),\n", - " (1, 0, 0.22452371802884058),\n", - " (1, 1, 0.4062437017193898),\n", - " (2, 0, 0.12135363071412639),\n", - " (2, 1, 0.21957211732999163),\n", - " (3, 0, 0.38530591042420903),\n", - " (3, 1, 0.6971561878597784),\n", - " (4, 0, 0.0950410525952534),\n", - " (4, 1, 0.17196325341736698),\n", - " (5, 0, 0.001130170114032713),\n", - " (5, 1, 0.002044881915942166)]" + "[(0, 0, 0.39281429109728644),\n", + " (0, 1, 0.11084571763784679),\n", + " (1, 0, 0.31553916068405957),\n", + " (1, 1, 0.08903994966976915),\n", + " (2, 0, 0.056793133938106345),\n", + " (2, 1, 0.016026086196320767),\n", + " (3, 0, 0.17125415796122967),\n", + " (3, 1, 0.048325100353786),\n", + " (4, 0, 0.5258110538125303),\n", + " (4, 1, 0.14837521170360757),\n", + " (5, 0, 0.4570556207765081),\n", + " (5, 1, 0.12897356189323625)]" ] }, "execution_count": 11, @@ -457,26 +459,26 @@ { "data": { "text/plain": [ - "[((0, 0), 2.8276956385710266),\n", - " ((0, 1), 4.344623698461026),\n", - " ((1, 0), 2.709031589561176),\n", - " ((1, 1), 4.286096753861166),\n", - " ((2, 0), 2.319702595813269),\n", - " ((2, 1), 4.014868661312692),\n", - " ((3, 0), 2.614588815326311),\n", - " ((3, 1), 4.66117541697328),\n", - " ((4, 0), 1.9932138244874387),\n", - " ((4, 1), 3.4969031578212193),\n", - " ((5, 0), 3.0858087055402157),\n", - " ((5, 1), 3.678401085869831),\n", - " ((6, 0), 2.9980017336305025),\n", - " ((6, 1), 4.739726686558273),\n", - " ((7, 0), 3.5599644578749863),\n", - " ((7, 1), 4.859619515198687),\n", - " ((8, 0), 2.515794511441488),\n", - " ((8, 1), 3.26530604000568),\n", - " ((9, 0), 2.218822328336375),\n", - " ((9, 1), 3.7604113556777747)]" + "[((0, 0), 2.2910294615369833),\n", + " ((0, 1), 3.5384779259752115),\n", + " ((1, 0), 1.588508508773101),\n", + " ((1, 1), 2.7899938642491398),\n", + " ((2, 0), 1.400567677321856),\n", + " ((2, 1), 3.7188621947679996),\n", + " ((3, 0), 1.4587357182350686),\n", + " ((3, 1), 3.4972992791592126),\n", + " ((4, 0), 2.24052580041269),\n", + " ((4, 1), 3.470907222987571),\n", + " ((5, 0), 1.1513307163420712),\n", + " ((5, 1), 1.7900281634340787),\n", + " ((6, 0), 1.454002494006837),\n", + " ((6, 1), 3.658028135367462),\n", + " ((7, 0), 2.552136746927374),\n", + " ((7, 1), 2.8249568885507244),\n", + " ((8, 0), 2.0401537352713808),\n", + " ((8, 1), 2.7811840133848817),\n", + " ((9, 0), 2.6432220797579706),\n", + " ((9, 1), 4.468515860388137)]" ] }, "execution_count": 12, @@ -508,16 +510,16 @@ { "data": { "text/plain": [ - "array([[2.82769564, 4.3446237 ],\n", - " [2.70903159, 4.28609675],\n", - " [2.3197026 , 4.01486866],\n", - " [2.61458882, 4.66117542],\n", - " [1.99321382, 3.49690316],\n", - " [3.08580871, 3.67840109],\n", - " [2.99800173, 4.73972669],\n", - " [3.55996446, 4.85961952],\n", - " [2.51579451, 3.26530604],\n", - " [2.21882233, 3.76041136]])" + "array([[2.29102946, 3.53847793],\n", + " [1.58850851, 2.78999386],\n", + " [1.40056768, 3.71886219],\n", + " [1.45873572, 3.49729928],\n", + " [2.2405258 , 3.47090722],\n", + " [1.15133072, 1.79002816],\n", + " [1.45400249, 3.65802814],\n", + " [2.55213675, 2.82495689],\n", + " [2.04015374, 2.78118401],\n", + " [2.64322208, 4.46851586]])" ] }, "execution_count": 13, @@ -544,7 +546,18 @@ "cell_type": "code", "execution_count": 14, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/xadupre/vv/this312/lib/python3.12/site-packages/pyspark/sql/pandas/conversion.py:676: FutureWarning: PySpark does not yet fully support pandas >= 3.0.0. Some features may not work correctly. It is recommended to use pandas < 3.0.0 for now.\n", + " require_minimum_pandas_version()\n", + "/home/xadupre/vv/this312/lib/python3.12/site-packages/pyspark/sql/pandas/conversion.py:953: FutureWarning: PySpark does not yet fully support pandas >= 3.0.0. Some features may not work correctly. It is recommended to use pandas < 3.0.0 for now.\n", + " require_minimum_pandas_version()\n" + ] + } + ], "source": [ "schema = [\"index\"] + [\"c%d\" % i for i in range(1, 11)]\n", "mat1 = spark.createDataFrame(\n", @@ -697,6 +710,22 @@ "execution_count": 21, "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/xadupre/vv/this312/lib/python3.12/site-packages/pyspark/sql/pandas/conversion.py:324: FutureWarning: PySpark does not yet fully support pandas >= 3.0.0. Some features may not work correctly. It is recommended to use pandas < 3.0.0 for now.\n", + " require_minimum_pandas_version()\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/xadupre/vv/this312/lib/python3.12/site-packages/pyspark/sql/pandas/conversion.py:324: FutureWarning: PySpark does not yet fully support pandas >= 3.0.0. Some features may not work correctly. It is recommended to use pandas < 3.0.0 for now.\n", + " require_minimum_pandas_version()\n" + ] + }, { "data": { "text/plain": [ @@ -786,7 +815,8 @@ "name": "stderr", "output_type": "stream", "text": [ - " \r" + "/home/xadupre/vv/this312/lib/python3.12/site-packages/pyspark/sql/pandas/conversion.py:324: FutureWarning: PySpark does not yet fully support pandas >= 3.0.0. Some features may not work correctly. It is recommended to use pandas < 3.0.0 for now.\n", + " require_minimum_pandas_version()\n" ] }, { @@ -823,46 +853,46 @@ " 0\n", " 0\n", " 0\n", - " 0.036595\n", + " 0.427635\n", " 0\n", - " 0\n", - " 0.452410\n", + " 1\n", + " 0.259207\n", " \n", " \n", " 1\n", " 0\n", " 0\n", - " 0.036595\n", + " 0.427635\n", " 0\n", - " 1\n", - " 0.818571\n", + " 0\n", + " 0.918575\n", " \n", " \n", " 2\n", - " 1\n", - " 0\n", - " 0.496284\n", " 0\n", - " 0\n", - " 0.452410\n", + " 1\n", + " 0.710440\n", + " 1\n", + " 1\n", + " 0.395866\n", " \n", " \n", " 3\n", - " 1\n", - " 0\n", - " 0.496284\n", " 0\n", " 1\n", - " 0.818571\n", + " 0.710440\n", + " 1\n", + " 0\n", + " 0.123740\n", " \n", " \n", " 4\n", - " 2\n", - " 0\n", - " 0.268238\n", " 0\n", - " 0\n", - " 0.452410\n", + " 2\n", + " 0.556229\n", + " 2\n", + " 1\n", + " 0.715428\n", " \n", " \n", "\n", @@ -870,11 +900,11 @@ ], "text/plain": [ " index pos col index2 pos2 col2\n", - "0 0 0 0.036595 0 0 0.452410\n", - "1 0 0 0.036595 0 1 0.818571\n", - "2 1 0 0.496284 0 0 0.452410\n", - "3 1 0 0.496284 0 1 0.818571\n", - "4 2 0 0.268238 0 0 0.452410" + "0 0 0 0.427635 0 1 0.259207\n", + "1 0 0 0.427635 0 0 0.918575\n", + "2 0 1 0.710440 1 1 0.395866\n", + "3 0 1 0.710440 1 0 0.123740\n", + "4 0 2 0.556229 2 1 0.715428" ] }, "execution_count": 25, @@ -925,7 +955,16 @@ "cell_type": "code", "execution_count": 28, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/xadupre/vv/this312/lib/python3.12/site-packages/pyspark/sql/pandas/conversion.py:324: FutureWarning: PySpark does not yet fully support pandas >= 3.0.0. Some features may not work correctly. It is recommended to use pandas < 3.0.0 for now.\n", + " require_minimum_pandas_version()\n" + ] + } + ], "source": [ "df = final.toPandas()" ] @@ -963,46 +1002,46 @@ " \n", " \n", " \n", - " 7\n", + " 0\n", " 0\n", " 0\n", - " 2.827696\n", + " 2.291029\n", " \n", " \n", - " 10\n", + " 1\n", " 0\n", " 1\n", - " 4.344624\n", + " 3.538478\n", " \n", " \n", - " 18\n", + " 3\n", " 1\n", " 0\n", - " 2.709032\n", + " 1.588509\n", " \n", " \n", - " 3\n", + " 2\n", " 1\n", " 1\n", - " 4.286097\n", + " 2.789994\n", " \n", " \n", - " 6\n", + " 4\n", " 2\n", " 0\n", - " 2.319703\n", + " 1.400568\n", " \n", " \n", "\n", "" ], "text/plain": [ - " i j sum(val)\n", - "7 0 0 2.827696\n", - "10 0 1 4.344624\n", - "18 1 0 2.709032\n", - "3 1 1 4.286097\n", - "6 2 0 2.319703" + " i j sum(val)\n", + "0 0 0 2.291029\n", + "1 0 1 3.538478\n", + "3 1 0 1.588509\n", + "2 1 1 2.789994\n", + "4 2 0 1.400568" ] }, "execution_count": 29, @@ -1060,11 +1099,16 @@ "source": [ "spark.stop()" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] } ], "metadata": { "kernelspec": { - "display_name": "this312", + "display_name": "this312 (3.12.3)", "language": "python", "name": "python3" }, From c335adc79b2113624e2c8989300838dd8a61b968 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= Date: Wed, 4 Nov 2026 09:44:33 +0100 Subject: [PATCH 3/6] fix --- _doc/articles/2026-11-04-technique.rst | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/_doc/articles/2026-11-04-technique.rst b/_doc/articles/2026-11-04-technique.rst index c3980ca..d3d9018 100644 --- a/_doc/articles/2026-11-04-technique.rst +++ b/_doc/articles/2026-11-04-technique.rst @@ -1,8 +1,32 @@ 2026-11-04 - une technique pour chaque échelle, chaque problème +**Plan** + +* Partie 1 : voyage technique d'un POC jusqu'à une grosse entreprise de données +* Partie 2 : quelques cas pratiques + +**Cas pratiques** + +* Monter un petit dashboard avec github +* Comment le faire sans gituhb ? +* Spark et Map Reduce, une logique différente pour écrire des calculs + +**L'IA ou les agents** + +Qu'est ce qui change ? + +A-t-on encore besoin d'un outil comme Grafana ? +- POC +- Production ? +- Dépendance ? **De la startup acoustique à la plateforme mondiale de données** +- des données qu'on collecte +- des données qu'on produit avec les outils collectant les données (pannes, ...) +- lire et écrire en même temps +- pandas / polars +- ClickHouse / PostgreSQL 30 étapes de développement technologique et industriel. From c0f0f209b890c9722de32031860b281d73032a3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= Date: Wed, 4 Nov 2026 11:21:07 +0100 Subject: [PATCH 4/6] Fix PR CI dependencies, Windows wheel repair and article title Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- _doc/articles/2026-11-04-technique.rst | 4 +++- pyproject.toml | 4 ++++ requirements-dev.txt | 3 +++ setup.py | 30 ++++++++++++++++---------- 4 files changed, 29 insertions(+), 12 deletions(-) diff --git a/_doc/articles/2026-11-04-technique.rst b/_doc/articles/2026-11-04-technique.rst index d3d9018..361e680 100644 --- a/_doc/articles/2026-11-04-technique.rst +++ b/_doc/articles/2026-11-04-technique.rst @@ -1,4 +1,5 @@ 2026-11-04 - une technique pour chaque échelle, chaque problème +=============================================================== **Plan** @@ -16,6 +17,7 @@ Qu'est ce qui change ? A-t-on encore besoin d'un outil comme Grafana ? + - POC - Production ? - Dépendance ? @@ -153,7 +155,7 @@ A-t-on encore besoin d'un outil comme Grafana ? Capacités transversales -======================= +----------------------- - Corrélation spatio-temporelle des observations de capteurs. - Fusion de données multi-capteurs (bruit, température, fumée, vibrations, diff --git a/pyproject.toml b/pyproject.toml index 3923e8b..3703064 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -155,6 +155,10 @@ skip = "pypy* pp* cp36-* cp37-* cp38-* cp39-* cp310-* cp314-* cp314t-* cp315-*" archs = ["AMD64"] build = "cp*" skip = "cp36-* cp37-* cp38-* cp39-* cp314-* cp314t-* cp315-* pypy*" +# PyTorch loads its own DLLs before importing the extension. +repair-wheel-command = "delvewheel repair -w {dest_dir} -v --exclude c10.dll;torch.dll;torch_cpu.dll;torch_python.dll {wheel}" +test-requires = ["torch"] +test-command = "python -c \"import teachcompute;teachcompute.check_installation(val=True)\" && python {project}/_unittests/ut_torch_extensions/test_fcts_piecewise_linear.py" [tool.cython-lint] max-line-length = 88 diff --git a/requirements-dev.txt b/requirements-dev.txt index 5e2397d..de79110 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,6 @@ +--find-links https://github.com/xadupre/onnx-light/releases/expanded_assets/0.2.6 +# Select the full wheel rather than the reduced build on Linux CI. +onnx-light @ https://github.com/xadupre/onnx-light/releases/download/0.2.6/onnx_light-0.2.6-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl ; python_version == "3.12" and sys_platform == "linux" and platform_machine == "x86_64" black black-nb clang-format diff --git a/setup.py b/setup.py index 5079ac9..f26c2dc 100644 --- a/setup.py +++ b/setup.py @@ -215,15 +215,19 @@ class cmake_build_class_extension(Command): ( "use-cuda=", None, - "If cuda is available, CUDA is " - "used by default unless this option is set to 0", + ( + "If cuda is available, CUDA is " + "used by default unless this option is set to 0" + ), ), ("use-nvtx=", None, "Enables compilation with NVTX events."), ( "cuda-version=", None, - "If cuda is available, it searches the installed version " - "unless this option is defined.", + ( + "If cuda is available, it searches the installed version " + "unless this option is defined." + ), ), ( "parallel=", @@ -233,17 +237,21 @@ class cmake_build_class_extension(Command): ( "cuda-build=", None, - "CUDA code can be compiled to be working with " - "different architectures, this flag can optimize " - "for a specific machine, possible values: DEFAULT, " - "H100, H100opt", + ( + "CUDA code can be compiled to be working with " + "different architectures, this flag can optimize " + "for a specific machine, possible values: DEFAULT, " + "H100, H100opt" + ), ), ( "cuda-link=", None, - "CUDA can statically linked (STATIC) or dynamically " - "(SHARED), default is STATIC." - "STATIC", + ( + "CUDA can statically linked (STATIC) or dynamically " + "(SHARED), default is STATIC." + "STATIC" + ), ), ( "cuda-nvcc=", From c7115e1391ad0d10d53a8e67225afeced0782ede Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= Date: Wed, 4 Nov 2026 11:42:05 +0100 Subject: [PATCH 5/6] Avoid probing the CUDA driver during native build discovery Resolve PyTorch headers and libraries without importing cpp_extension, whose CUDA availability probe can crash before CMake starts. Add a regression test that rejects CUDA driver probes during build path discovery. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- _unittests/ut_xrun_doc/test_setup.py | 48 ++++++++++++++++++++++++++++ setup.py | 14 ++++++-- 2 files changed, 59 insertions(+), 3 deletions(-) create mode 100644 _unittests/ut_xrun_doc/test_setup.py diff --git a/_unittests/ut_xrun_doc/test_setup.py b/_unittests/ut_xrun_doc/test_setup.py new file mode 100644 index 0000000..0e31138 --- /dev/null +++ b/_unittests/ut_xrun_doc/test_setup.py @@ -0,0 +1,48 @@ +import subprocess +import sys +import unittest +from pathlib import Path + + +class TestSetup(unittest.TestCase): + def test_torch_paths_without_cuda_probe(self): + setup = Path(__file__).resolve().parents[2] / "setup.py" + script = """ +import sys +from pathlib import Path +import setuptools +from distutils.core import run_setup +import torch + + +def unexpected_cuda_probe(): + raise AssertionError("Finding build paths must not probe the CUDA driver") + + +torch.cuda.is_available = unexpected_cuda_probe +distribution = run_setup(sys.argv[1], stop_after="config") +command = distribution.get_command_obj("build_ext") +command.use_cuda = False +command.ensure_finalized() +definitions = dict( + argument[2:].split("=", 1) + for argument in command.get_cmake_args("Release") + if argument.startswith("-D") +) +includes = [Path(path) for path in definitions["TORCH_INCLUDE"].split(";")] +for header in ["torch/extension.h", "torch/torch.h"]: + assert any((path / header).is_file() for path in includes), header +assert Path(definitions["TORCH_LIBRARIES_DIR"]).is_dir() +assert Path(definitions["CMAKE_PREFIX_PATH"]).is_dir() +""" + result = subprocess.run( + [sys.executable, "-c", script, str(setup)], + cwd=setup.parent, + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/setup.py b/setup.py index f26c2dc..b9df50f 100644 --- a/setup.py +++ b/setup.py @@ -419,10 +419,18 @@ def get_cmake_args(self, cfg: str) -> List[str]: try: import torch.utils - from torch.utils.cpp_extension import include_paths, library_paths - torch_include = ";".join(include_paths()) - torch_libraries = ";".join(library_paths()) + # cpp_extension probes the CUDA driver even for CPU header paths. + torch_dir = os.path.dirname(torch.__file__) + torch_include = ";".join( + [ + os.path.join(torch_dir, "include"), + os.path.join( + torch_dir, "include", "torch", "csrc", "api", "include" + ), + ] + ) + torch_libraries = os.path.join(torch_dir, "lib") torch_prefix = torch.utils.cmake_prefix_path except ImportError: torch_include = None From a3389a1472272157c045e8f8497fadfd9454c844 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= Date: Wed, 4 Nov 2026 12:42:37 +0100 Subject: [PATCH 6/6] Exclude unsupported Python 3.15 free-threaded wheel builds The existing cp315-* exclusion does not match cp315t-* identifiers. Exclude the free-threaded variant consistently across platforms while preserving the supported wheel matrix, verified using cibuildwheel 4.3.5 from CI. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3703064..b626cb3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -129,7 +129,7 @@ test-command = "python -c \"import teachcompute;teachcompute.check_installation( [tool.cibuildwheel.linux] archs = ["x86_64"] build = "cp*" -skip = "cp36-* cp37-* cp38-* cp39-* cp314-* cp314t-* cp315-* pypy* *musllinux*" +skip = "cp36-* cp37-* cp38-* cp39-* cp314-* cp314t-* cp315-* cp315t-* pypy* *musllinux*" manylinux-x86_64-image = "manylinux_2_28" before-build = "pip install auditwheel-symbols abi3audit;python -c 'import sysconfig;print(sysconfig.get_platform())'" build-verbosity = 1 @@ -148,13 +148,13 @@ environment = """ """ build = "cp*" before-build = "brew install llvm libomp" -skip = "pypy* pp* cp36-* cp37-* cp38-* cp39-* cp310-* cp314-* cp314t-* cp315-*" +skip = "pypy* pp* cp36-* cp37-* cp38-* cp39-* cp310-* cp314-* cp314t-* cp315-* cp315t-*" # environment="LDFLAGS='-L/opt/homebrew/opt/llvm/lib -L/opt/homebrew/opt/libomp/lib' CPPFLAGS='-I/opt/homebrew/opt/llvm/include -I/opt/homebrew/opt/libomp/include'" [tool.cibuildwheel.windows] archs = ["AMD64"] build = "cp*" -skip = "cp36-* cp37-* cp38-* cp39-* cp314-* cp314t-* cp315-* pypy*" +skip = "cp36-* cp37-* cp38-* cp39-* cp314-* cp314t-* cp315-* cp315t-* pypy*" # PyTorch loads its own DLLs before importing the extension. repair-wheel-command = "delvewheel repair -w {dest_dir} -v --exclude c10.dll;torch.dll;torch_cpu.dll;torch_python.dll {wheel}" test-requires = ["torch"]