@inproceedings{Voogd2023ReinforcementLF,

title = {Reinforcement Learning from Simulation to Real World Autonomous Driving using Digital Twin},

author = {Kevin Voogd and Jean Pierre Allamaa and Javier Alonso-Mora and Tong Duy Son},

url = {https://www.sciencedirect.com/science/article/pii/S2405896323022553},

doi = {https://doi.org/10.1016/j.ifacol.2023.10.1846},

issn = {2405-8963},

year  = {2023},

date = {2023-11-30},

urldate = {2023-11-30},

booktitle = {22nd IFAC World Congress 2023},

journal = {IFAC-PapersOnLine},

volume = {56},

number = {2},

pages = {1510-1515},

publisher = {Elsevier Ltd},

address = {Yokohama, Japan},

abstract = {Reinforcement learning (RL) is a promising solution for autonomous vehicles to deal with complex and uncertain traffic environments. The RL training process is however expensive, unsafe, and time-consuming. Algorithms are often developed first in simulation and then transferred to the real-world, leading to a common sim2real challenge where performance decreases when the domain changes. In this paper, we propose a transfer learning process to minimize the gap by exploiting digital twin technology, relying on a systematic and simultaneous combination of virtual and real world data coming from vehicle dynamics and traffic scenarios. The model and testing environment is evolved from model, hardware to vehicle in the loop and proving ground testing stages, similar to standard development cycle in the automotive industry. In particular, we also integrate other transfer learning techniques such as domain randomization and adaptation in each stage. The simulation and real data are gradually incorporated to accelerate and make the transfer learning process more robust. The proposed RL methodology is applied to develop a path-following steering controller for an autonomous electric vehicle. After learning and deploying the real-time RL control policy on the vehicle, we obtained satisfactory and safe control performance already from the first deployment, demonstrating the advantages of the proposed digital twin based learning process.

},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{Allamaa2022SafetyEF,

title = {Safety Envelope for Orthogonal Collocation Methods in Embedded Optimal Control},

author = {Jean Pierre Allamaa and Panagiotis Patrinos and Herman Van Auweraer and Tong Duy Son},

url = {https://ieeexplore.ieee.org/document/10178116

https://arxiv.org/abs/2211.14853},

doi = {https://doi.org/10.23919/ECC57647.2023.10178116},

isbn = {978-3-907144-08-4},

year  = {2023},

date = {2023-07-17},

urldate = {2023-07-17},

booktitle = {2023 European Control Conference (ECC)},

pages = {1-7},

publisher = {IEEE},

address = {Bucharest, Romania},

abstract = {Orthogonal collocation methods are direct approaches for solving optimal control problems (OCP). A high solution accuracy is achieved with few optimization variables, making it more favorable for embedded and real-time NMPC applications. However, collocation approaches lack a guarantee about the safety of the resulting trajectory as inequality constraints are only set on a finite number of collocation points. In this paper we propose a method to efficiently create a convex safety envelope containing the trajectory such that the solution fully satisfies the OCP constraints. We make use of Bernstein approximations of a polynomial’s extrema and span the solution over an orthogonal basis using Legendre polynomials. The tightness of the safety envelope estimation, high accuracy in solving the underlying differential equations, fast rate of convergence and little conservatism are properties of the presented approach making it a suitable method for safe real-time NMPC deployment. We show that our method has comparable computational performance to pseudospectral approaches and can accurately approximate the original OCP up to 9 times more quickly than standard multiple-shooting method in autonomous driving applications, without adding complexity to the formulation.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{Allamaa2022RTNMPCStrategy,

title = {Real-time Nonlinear MPC Strategy with Full Vehicle Validation for Autonomous Driving},

author = {Jean Pierre Allamaa and Petr Listov and Herman Van Auweraer and Colin Jones and Tong Duy Son},

url = {https://ieeexplore.ieee.org/document/9867514},

doi = {10.23919/ACC53348.2022.9867514},

isbn = {978-1-6654-5196-3},

year  = {2022},

date = {2022-06-09},

booktitle = {2022 American Control Conference (ACC)},

pages = {1982-1987},

publisher = {IEEE},

address = {Atlanta, GA, USA},

abstract = {In this paper, we present the development and deployment of an embedded optimal control strategy for autonomous driving applications on a Ford Focus road vehicle. Non-linear model predictive control (NMPC) is designed and deployed on a system with hard real-time constraints. We show the properties of sequential quadratic programming (SQP) optimization solvers that are suitable for driving tasks. Importantly, the designed algorithms are validated based on a standard automotive XiL development cycle: model-in-the-loop (MiL) with high fidelity vehicle dynamics, hardware-in-the-loop (HiL) with vehicle actuation and embedded platform, and full vehicle-hardware-in-the-loop (VeHiL). The autonomous driving environment contains both virtual simulation and physical proving ground tracks. NMPC algorithms and optimal control problem formulation are fine-tuned using a deployable C code via code generation compatible with the target embedded toolchains. Finally, the developed systems are applied to autonomous collision avoidance, trajectory tracking, and lane change at high speed on city/highway and low speed at a parking environment.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{ALLAMAA2022385,

title = {Sim2real for Autonomous Vehicle Control using Executable Digital Twin},

author = {Jean Pierre Allamaa and Panagiotis Patrinos and Herman Van Auweraer and Tong Duy Son},

url = {https://www.sciencedirect.com/science/article/pii/S2405896322023461},

doi = {https://doi.org/10.1016/j.ifacol.2022.10.314},

issn = {2405-8963},

year  = {2022},

date = {2022-01-01},

urldate = {2022-01-01},

booktitle = {10th IFAC Symposium on Advances in Automotive Control (AAC)},

journal = {IFAC-PapersOnLine},

volume = {55},

number = {24},

pages = {385-391},

publisher = {Elsevier Ltd},

abstract = {In this work, we propose a sim2real method to transfer and adapt a nonlinear model predictive controller (NMPC) from simulation to the real target system based on executable digital twin (xDT). The xDT model is a high fidelity vehicle dynamics simulator, executable online in the control parameter randomization and learning process. The parameters are adapted to gradually improve control performance and deal with changing real-world environment. In particular, the performance metric is not required to be differentiable nor analytical with respect to the control parameters and system dynamics are not necessary linearized. Eventually, the proposed sim2real framework leverages altogether online high fidelity simulator, data-driven estimations, and simulation based optimization to transfer and adapt efficiently a controller developed in simulation environment to the real platform. Our experiment demonstrates that a high control performance is achieved without tedious time and labor consuming tuning.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}