[{"data":1,"prerenderedAt":1127},["ShallowReactive",2],{"Categories":3,"NavIndexCategoriesCountFooter":203,"content-\u002F2025\u002F05\u002F17\u002Frecovering-longhorn-backups-after-a-k3s-control-plane-crash\u002F":204},[4,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,68,70,71,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202],{"category":5},"System Administration",{"category":5},{"category":5},{"category":5},{"category":5},{"category":5},{"category":5},{"category":5},{"category":5},{"category":5},{"category":5},{"category":5},{"category":5},{"category":5},{"category":5},{"category":5},{"category":5},{"category":5},{"category":5},{"category":5},{"category":5},{"category":27},"Software Development",{"category":5},{"category":5},{"category":5},{"category":5},{"category":27},{"category":27},{"category":5},{"category":5},{"category":5},{"category":27},{"category":5},{"category":5},{"category":5},{"category":27},{"category":27},{"category":27},{"category":27},{"category":5},{"category":5},{"category":5},{"category":27},{"category":27},{"category":5},{"category":5},{"category":5},{"category":5},{"category":5},{"category":5},{"category":27},{"category":5},{"category":5},{"category":27},{"category":27},{"category":27},{"category":27},{"category":5},{"category":27},{"category":27},{"category":67},"Drones & RC",{"category":69},"DIY Projects",{"category":67},{"category":72},"Photography",{"category":69},{"category":69},{"category":69},{"category":67},{"category":69},{"category":69},{"category":69},{"category":69},{"category":69},{"category":69},{"category":69},{"category":69},{"category":69},{"category":69},{"category":69},{"category":69},{"category":69},{"category":69},{"category":67},{"category":69},{"category":69},{"category":67},{"category":67},{"category":72},{"category":72},{"category":72},{"category":67},{"category":67},{"category":67},{"category":67},{"category":67},{"category":67},{"category":67},{"category":67},{"category":67},{"category":67},{"category":5},{"category":5},{"category":72},{"category":67},{"category":67},{"category":67},{"category":67},{"category":67},{"category":67},{"category":5},{"category":67},{"category":67},{"category":72},{"category":72},{"category":67},{"category":67},{"category":67},{"category":67},{"category":67},{"category":67},{"category":67},{"category":67},{"category":67},{"category":67},{"category":67},{"category":67},{"category":72},{"category":67},{"category":138},"3D Printing - Laser Cutting - CNC",{"category":138},{"category":138},{"category":138},{"category":138},{"category":138},{"category":138},{"category":138},{"category":138},{"category":138},{"category":138},{"category":138},{"category":5},{"category":138},{"category":27},{"category":27},{"category":138},{"category":138},{"category":72},{"category":158},"Photography,3D Printing - Laser Cutting - CNC",{"category":27},{"category":27},{"category":69},{"category":27},{"category":27},{"category":27},{"category":27},{"category":5},{"category":67},{"category":5},{"category":5},{"category":27},{"category":27},{"category":27},{"category":27},{"category":27},{"category":69},{"category":27},{"category":27},{"category":27},{"category":27},{"category":181},"Home Assistant",{"category":181},{"category":72},{"category":27},{"category":27},{"category":72},{"category":138},{"category":5},{"category":72},{"category":72},{"category":138},{"category":27},{"category":181},{"category":181},{"category":72},{"category":72},{"category":72},{"category":72},{"category":72},{"category":72},{"category":72},{"category":72},191,{"id":205,"title":206,"body":207,"category":1111,"date":1112,"description":1113,"embedImage":1111,"extension":1114,"image":1111,"intro":1115,"meta":1116,"navigation":1117,"path":1118,"seo":1119,"series":1111,"sitemap":1120,"stem":1121,"tags":1122,"__hash__":1126},"content\u002F2025\u002F05\u002F17\u002Frecovering-longhorn-backups-after-a-k3s-control-plane-crash.md","Recovering longhorn backups after a k3s control-plane crash",{"type":208,"value":209,"toc":1101},"minimark",[210,225,228,243,246,249,252,257,262,265,281,316,320,323,350,353,427,438,442,445,453,456,467,471,474,477,579,582,593,599,602,605,608,611,614,771,774,877,880,885,888,891,895,898,901,1075,1078,1082,1085,1088,1091,1094,1097],[211,212,213,214,224],"p",{},"I run a little ",[215,216,223],"a",{"href":217,"rel":218,"target":222},"https:\u002F\u002Fdocs.k3s.io\u002F",[219,220,221],"nofollow","noopener","noreferer","_blank","k3s"," cluster on a couple of Raspberry Pi 5s and an old intel nuc at home.",[211,226,227],{},"It doesn't run anything too important but it allows me to play with kubernetes and to work with systems that are in\ncombined intel\u002Farm64 environments.",[211,229,230,231,236,237,242],{},"The cluster was deployed with ",[215,232,235],{"href":233,"rel":234,"target":222},"https:\u002F\u002Ffluxcd.io\u002Fflux\u002F",[219,220,221],"fluxcd",", ",[215,238,241],{"href":239,"rel":240,"target":222},"https:\u002F\u002Flonghorn.io\u002Fdocs",[219,220,221],"longhorn storage"," using\nlocal disk, with longhorn backup to a NAS share over CIFS\u002FSamba.",[211,244,245],{},"To avoid the pi's eating the SD cards I run them using SSD instead of SD cards.",[211,247,248],{},"The SSD on the control-plane pi simply died.",[211,250,251],{},"Given that there was no data not in flux apart from longhorn - the backup strategy was the flux repo and the longhorn\nbackups.",[253,254,256],"h2",{"id":255},"the-rebuild","The rebuild",[258,259,261],"h3",{"id":260},"rebuilding-the-os","Rebuilding the OS",[211,263,264],{},"This is simple",[266,267,268,272,275,278],"ul",{},[269,270,271],"li",{},"default install of the OS to the SSD (raspian 64 bit no desktop)",[269,273,274],{},"boot - allow system to do first time boot setup",[269,276,277],{},"run ansible - this installs everything on the pi apart from k3s",[269,279,280],{},"edit \u002Fboot\u002Ffirmware\u002Fcmdline.txt to add the following to the end of the line (and reboot):",[282,283,288],"pre",{"className":284,"code":285,"language":286,"meta":287,"style":287},"language-shell shiki shiki-themes github-dark","cgroup_memory=1 cgroup_enable=memory\n","shell","",[289,290,291],"code",{"__ignoreMap":287},[292,293,296,300,304,308,311,313],"span",{"class":294,"line":295},"line",1,[292,297,299],{"class":298},"s95oV","cgroup_memory",[292,301,303],{"class":302},"snl16","=",[292,305,307],{"class":306},"sU2Wk","1",[292,309,310],{"class":298}," cgroup_enable",[292,312,303],{"class":302},[292,314,315],{"class":306},"memory\n",[258,317,319],{"id":318},"installing-k3s","Installing k3s",[211,321,322],{},"Standard - run:",[282,324,326],{"className":284,"code":325,"language":286,"meta":287,"style":287},"curl -sfL https:\u002F\u002Fget.k3s.io | sh -\n",[289,327,328],{"__ignoreMap":287},[292,329,330,334,338,341,344,347],{"class":294,"line":295},[292,331,333],{"class":332},"svObZ","curl",[292,335,337],{"class":336},"sDLfK"," -sfL",[292,339,340],{"class":306}," https:\u002F\u002Fget.k3s.io",[292,342,343],{"class":302}," |",[292,345,346],{"class":332}," sh",[292,348,349],{"class":306}," -\n",[211,351,352],{},"On the two agents - run:",[282,354,356],{"className":284,"code":355,"language":286,"meta":287,"style":287},"curl -sfL https:\u002F\u002Fget.k3s.io | K3S_URL=https:\u002F\u002F\u003CIP of control-plane-node>:6443 K3S_TOKEN=\u003Ccontents of \u002Fvar\u002Flib\u002Francher\u002Fk3s\u002Fserver\u002Fnode-token from control-plane node> sh -\n",[289,357,358],{"__ignoreMap":287},[292,359,360,362,364,366,368,371,373,376,379,382,385,388,391,394,397,400,402,405,407,410,413,416,419,421,423,425],{"class":294,"line":295},[292,361,333],{"class":332},[292,363,337],{"class":336},[292,365,340],{"class":306},[292,367,343],{"class":302},[292,369,370],{"class":298}," K3S_URL",[292,372,303],{"class":302},[292,374,375],{"class":306},"https:\u002F\u002F",[292,377,378],{"class":302},"\u003C",[292,380,381],{"class":306},"IP",[292,383,384],{"class":332}," of",[292,386,387],{"class":306}," control-plane-nod",[292,389,390],{"class":298},"e",[292,392,393],{"class":302},">",[292,395,396],{"class":306},":6443",[292,398,399],{"class":306}," K3S_TOKEN=",[292,401,378],{"class":302},[292,403,404],{"class":306},"contents",[292,406,384],{"class":306},[292,408,409],{"class":306}," \u002Fvar\u002Flib\u002Francher\u002Fk3s\u002Fserver\u002Fnode-token",[292,411,412],{"class":306}," from",[292,414,415],{"class":306}," control-plane",[292,417,418],{"class":306}," nod",[292,420,390],{"class":298},[292,422,393],{"class":302},[292,424,346],{"class":306},[292,426,349],{"class":306},[211,428,429,430,433,434,437],{},"Finally - update your local ",[289,431,432],{},".kube\u002Fconfig"," from ",[289,435,436],{},"\u002Fetc\u002Francher\u002Fk3s\u002Fk3s.yaml"," on the control-plane node.",[258,439,441],{"id":440},"fluxing","Fluxing",[211,443,444],{},"Since my flux setup was experimental - I wanted to tidy it. So - I created a new flux setup follwing the docs on github\nbootstrap from fluxcd.",[211,446,447,448,452],{},"After that - I copied over the parts I wanted to keep - ",[449,450,451],"strong",{},"NOT"," including anything that used longhorn storage.",[211,454,455],{},"Some other things that were tidied up:",[266,457,458,461,464],{},[269,459,460],{},"Separate some kusomizations to allow for dependency ordering - for example - it won't try to install cert-issuer using\nthe cert-manager.io\u002Fv1 apiVersion before the cert-manager helm chart is installed (since it is this that provides that\napi)",[269,462,463],{},"Replaced the cert-issuer http01 letsencrypt with a dns01 issuer using cloudflare (this allows me to have my internal\ndomains also on letsencrypt instead of using a local CA)",[269,465,466],{},"Updated all helm charts and docker images to the latest versions.",[258,468,470],{"id":469},"storage","Storage",[211,472,473],{},"So to the storage.",[211,475,476],{},"At this point - each of the apps that used longhorn storage had a stateful set that used a volume claim template like\nthis:",[282,478,482],{"className":479,"code":480,"language":481,"meta":287,"style":287},"language-yaml shiki shiki-themes github-dark","  volumeClaimTemplates:\n    - metadata:\n        name: data-volume\n      spec:\n        accessModes:\n          - ReadWriteMany\n        storageClass: longhorn\n        resources:\n          requests:\n            storage: 1Gi\n","yaml",[289,483,484,493,504,516,524,532,541,552,560,568],{"__ignoreMap":287},[292,485,486,490],{"class":294,"line":295},[292,487,489],{"class":488},"s4JwU","  volumeClaimTemplates",[292,491,492],{"class":298},":\n",[292,494,496,499,502],{"class":294,"line":495},2,[292,497,498],{"class":298},"    - ",[292,500,501],{"class":488},"metadata",[292,503,492],{"class":298},[292,505,507,510,513],{"class":294,"line":506},3,[292,508,509],{"class":488},"        name",[292,511,512],{"class":298},": ",[292,514,515],{"class":306},"data-volume\n",[292,517,519,522],{"class":294,"line":518},4,[292,520,521],{"class":488},"      spec",[292,523,492],{"class":298},[292,525,527,530],{"class":294,"line":526},5,[292,528,529],{"class":488},"        accessModes",[292,531,492],{"class":298},[292,533,535,538],{"class":294,"line":534},6,[292,536,537],{"class":298},"          - ",[292,539,540],{"class":306},"ReadWriteMany\n",[292,542,544,547,549],{"class":294,"line":543},7,[292,545,546],{"class":488},"        storageClass",[292,548,512],{"class":298},[292,550,551],{"class":306},"longhorn\n",[292,553,555,558],{"class":294,"line":554},8,[292,556,557],{"class":488},"        resources",[292,559,492],{"class":298},[292,561,563,566],{"class":294,"line":562},9,[292,564,565],{"class":488},"          requests",[292,567,492],{"class":298},[292,569,571,574,576],{"class":294,"line":570},10,[292,572,573],{"class":488},"            storage",[292,575,512],{"class":298},[292,577,578],{"class":306},"1Gi\n",[211,580,581],{},"This will create a longhorn persistent volume and persistent volume claim with default naming.",[211,583,584,585,588,589,592],{},"The PV name will look like ",[289,586,587],{},"pvc-\u003CGUID>"," and the PVC name will look like\n",[289,590,591],{},"\u003CvolumeClaimTemplates.metadata.name>-\u003Cmetadata.name>-\u003CID>",".",[211,594,595,596,592],{},"So - for example - the data-volume in the \"share\" app stateful set created a PVC called ",[289,597,598],{},"data-volume-share-0",[211,600,601],{},"Now - in the longhorn backup list - I could see the backups for each of these volumes.",[211,603,604],{},"I was also able to restore them using the longhorn UI - keeping the same names.",[211,606,607],{},"However - you need to be able to tell flux to connect to a specific volume rather than generating a new one.",[211,609,610],{},"This was simple enough - it required adding two new files to each kustomization. Note that in the following - it ignores\nthe Capacity - since the volume exists - but I set it in case of a subsequent restore where I don't restore a backup.",[211,612,613],{},"One for the PV:",[282,615,617],{"className":479,"code":616,"language":481,"meta":287,"style":287},"apiVersion: v1\nkind: PersistentVolume\nmetadata:\n  name: pvc-\u003CCORRECT GUID>\nspec:\n  capacity:\n    storage: \u003CCapacity>\n  accessModes:\n    - ReadWriteMany\n  storageClassName: longhorn\n  persistentVolumeReclaimPolicy: Retain\n  csi:\n    driver: driver.longhorn.io\n    volumeHandle: pvc-\u003CCORRECT GUID>\n  claimRef:\n    namespace: \u003CCORRECT NAMESPACE FOR THE APP>\n    name: \u003CCORRECT PVC NAME - for example data-volume-share-0>\n",[289,618,619,629,639,645,655,662,669,679,686,692,701,712,720,731,741,749,760],{"__ignoreMap":287},[292,620,621,624,626],{"class":294,"line":295},[292,622,623],{"class":488},"apiVersion",[292,625,512],{"class":298},[292,627,628],{"class":306},"v1\n",[292,630,631,634,636],{"class":294,"line":495},[292,632,633],{"class":488},"kind",[292,635,512],{"class":298},[292,637,638],{"class":306},"PersistentVolume\n",[292,640,641,643],{"class":294,"line":506},[292,642,501],{"class":488},[292,644,492],{"class":298},[292,646,647,650,652],{"class":294,"line":518},[292,648,649],{"class":488},"  name",[292,651,512],{"class":298},[292,653,654],{"class":306},"pvc-\u003CCORRECT GUID>\n",[292,656,657,660],{"class":294,"line":526},[292,658,659],{"class":488},"spec",[292,661,492],{"class":298},[292,663,664,667],{"class":294,"line":534},[292,665,666],{"class":488},"  capacity",[292,668,492],{"class":298},[292,670,671,674,676],{"class":294,"line":543},[292,672,673],{"class":488},"    storage",[292,675,512],{"class":298},[292,677,678],{"class":306},"\u003CCapacity>\n",[292,680,681,684],{"class":294,"line":554},[292,682,683],{"class":488},"  accessModes",[292,685,492],{"class":298},[292,687,688,690],{"class":294,"line":562},[292,689,498],{"class":298},[292,691,540],{"class":306},[292,693,694,697,699],{"class":294,"line":570},[292,695,696],{"class":488},"  storageClassName",[292,698,512],{"class":298},[292,700,551],{"class":306},[292,702,704,707,709],{"class":294,"line":703},11,[292,705,706],{"class":488},"  persistentVolumeReclaimPolicy",[292,708,512],{"class":298},[292,710,711],{"class":306},"Retain\n",[292,713,715,718],{"class":294,"line":714},12,[292,716,717],{"class":488},"  csi",[292,719,492],{"class":298},[292,721,723,726,728],{"class":294,"line":722},13,[292,724,725],{"class":488},"    driver",[292,727,512],{"class":298},[292,729,730],{"class":306},"driver.longhorn.io\n",[292,732,734,737,739],{"class":294,"line":733},14,[292,735,736],{"class":488},"    volumeHandle",[292,738,512],{"class":298},[292,740,654],{"class":306},[292,742,744,747],{"class":294,"line":743},15,[292,745,746],{"class":488},"  claimRef",[292,748,492],{"class":298},[292,750,752,755,757],{"class":294,"line":751},16,[292,753,754],{"class":488},"    namespace",[292,756,512],{"class":298},[292,758,759],{"class":306},"\u003CCORRECT NAMESPACE FOR THE APP>\n",[292,761,763,766,768],{"class":294,"line":762},17,[292,764,765],{"class":488},"    name",[292,767,512],{"class":298},[292,769,770],{"class":306},"\u003CCORRECT PVC NAME - for example data-volume-share-0>\n",[211,772,773],{},"And one for the PVC:",[282,775,777],{"className":479,"code":776,"language":481,"meta":287,"style":287},"apiVersion: v1\nkind: PersistentVolumeClaim\nmetadata:\n  name: \u003CCORRECT PVC NAME - for example data-volume-share-0>\n  namespace: \u003CCORRECT NAMESPACE FOR THE APP>\nspec:\n  accessModes:\n    - ReadWriteMany\n  storageClassName: longhorn\n  resources:\n    requests:\n      storage: \u003CCapacity>\n  volumeName: pvc-\u003CCORRECT GUID>\n",[289,778,779,787,796,802,810,819,825,831,837,845,852,859,868],{"__ignoreMap":287},[292,780,781,783,785],{"class":294,"line":295},[292,782,623],{"class":488},[292,784,512],{"class":298},[292,786,628],{"class":306},[292,788,789,791,793],{"class":294,"line":495},[292,790,633],{"class":488},[292,792,512],{"class":298},[292,794,795],{"class":306},"PersistentVolumeClaim\n",[292,797,798,800],{"class":294,"line":506},[292,799,501],{"class":488},[292,801,492],{"class":298},[292,803,804,806,808],{"class":294,"line":518},[292,805,649],{"class":488},[292,807,512],{"class":298},[292,809,770],{"class":306},[292,811,812,815,817],{"class":294,"line":526},[292,813,814],{"class":488},"  namespace",[292,816,512],{"class":298},[292,818,759],{"class":306},[292,820,821,823],{"class":294,"line":534},[292,822,659],{"class":488},[292,824,492],{"class":298},[292,826,827,829],{"class":294,"line":543},[292,828,683],{"class":488},[292,830,492],{"class":298},[292,832,833,835],{"class":294,"line":554},[292,834,498],{"class":298},[292,836,540],{"class":306},[292,838,839,841,843],{"class":294,"line":562},[292,840,696],{"class":488},[292,842,512],{"class":298},[292,844,551],{"class":306},[292,846,847,850],{"class":294,"line":570},[292,848,849],{"class":488},"  resources",[292,851,492],{"class":298},[292,853,854,857],{"class":294,"line":703},[292,855,856],{"class":488},"    requests",[292,858,492],{"class":298},[292,860,861,864,866],{"class":294,"line":714},[292,862,863],{"class":488},"      storage",[292,865,512],{"class":298},[292,867,678],{"class":306},[292,869,870,873,875],{"class":294,"line":722},[292,871,872],{"class":488},"  volumeName",[292,874,512],{"class":298},[292,876,654],{"class":306},[211,878,879],{},"No changes were needed in the stateful set or the volume claim template.",[211,881,882],{},[449,883,884],{},"HOWEVER",[211,886,887],{},"If I had installed the entire app then there was a strong chance that the stateful set would create some new storage\nwhile reconciling all of this.",[211,889,890],{},"I chose to install each app without the stateful set then once the PV and PVC were in place - added it back to the\nkustomization. This avoided having to scale down and up and tidy up etc.",[253,892,894],{"id":893},"other-fixes","Other fixes?",[211,896,897],{},"One - one of the apps had an update that moved its image from using root user to a non-root user. And the db on the\nlonghorn image was owned by root.",[211,899,900],{},"To fix this - I added a temporary pod to the namespace with this:",[282,902,904],{"className":479,"code":903,"language":481,"meta":287,"style":287},"apiVersion: v1\nkind: Pod\nmetadata:\n  name: debug-fix\n  namespace: \u003CCORRECT NAMESPACE FOR THE APP>\nspec:\n  containers:\n    - name: shell\n      image: busybox\n      command: [ \"sleep\", \"3600\" ]\n      volumeMounts:\n        - mountPath: \u002Fdata\n          name: data\n  volumes:\n    - name: data\n      persistentVolumeClaim:\n        claimName: \u003CCORRECT NAME OF THE PVC - for example data-volume-share-0>\n  restartPolicy: Never\n",[289,905,906,914,923,929,938,946,952,959,971,981,1000,1007,1020,1030,1037,1047,1054,1064],{"__ignoreMap":287},[292,907,908,910,912],{"class":294,"line":295},[292,909,623],{"class":488},[292,911,512],{"class":298},[292,913,628],{"class":306},[292,915,916,918,920],{"class":294,"line":495},[292,917,633],{"class":488},[292,919,512],{"class":298},[292,921,922],{"class":306},"Pod\n",[292,924,925,927],{"class":294,"line":506},[292,926,501],{"class":488},[292,928,492],{"class":298},[292,930,931,933,935],{"class":294,"line":518},[292,932,649],{"class":488},[292,934,512],{"class":298},[292,936,937],{"class":306},"debug-fix\n",[292,939,940,942,944],{"class":294,"line":526},[292,941,814],{"class":488},[292,943,512],{"class":298},[292,945,759],{"class":306},[292,947,948,950],{"class":294,"line":534},[292,949,659],{"class":488},[292,951,492],{"class":298},[292,953,954,957],{"class":294,"line":543},[292,955,956],{"class":488},"  containers",[292,958,492],{"class":298},[292,960,961,963,966,968],{"class":294,"line":554},[292,962,498],{"class":298},[292,964,965],{"class":488},"name",[292,967,512],{"class":298},[292,969,970],{"class":306},"shell\n",[292,972,973,976,978],{"class":294,"line":562},[292,974,975],{"class":488},"      image",[292,977,512],{"class":298},[292,979,980],{"class":306},"busybox\n",[292,982,983,986,989,992,994,997],{"class":294,"line":570},[292,984,985],{"class":488},"      command",[292,987,988],{"class":298},": [ ",[292,990,991],{"class":306},"\"sleep\"",[292,993,236],{"class":298},[292,995,996],{"class":306},"\"3600\"",[292,998,999],{"class":298}," ]\n",[292,1001,1002,1005],{"class":294,"line":703},[292,1003,1004],{"class":488},"      volumeMounts",[292,1006,492],{"class":298},[292,1008,1009,1012,1015,1017],{"class":294,"line":714},[292,1010,1011],{"class":298},"        - ",[292,1013,1014],{"class":488},"mountPath",[292,1016,512],{"class":298},[292,1018,1019],{"class":306},"\u002Fdata\n",[292,1021,1022,1025,1027],{"class":294,"line":722},[292,1023,1024],{"class":488},"          name",[292,1026,512],{"class":298},[292,1028,1029],{"class":306},"data\n",[292,1031,1032,1035],{"class":294,"line":733},[292,1033,1034],{"class":488},"  volumes",[292,1036,492],{"class":298},[292,1038,1039,1041,1043,1045],{"class":294,"line":743},[292,1040,498],{"class":298},[292,1042,965],{"class":488},[292,1044,512],{"class":298},[292,1046,1029],{"class":306},[292,1048,1049,1052],{"class":294,"line":751},[292,1050,1051],{"class":488},"      persistentVolumeClaim",[292,1053,492],{"class":298},[292,1055,1056,1059,1061],{"class":294,"line":762},[292,1057,1058],{"class":488},"        claimName",[292,1060,512],{"class":298},[292,1062,1063],{"class":306},"\u003CCORRECT NAME OF THE PVC - for example data-volume-share-0>\n",[292,1065,1067,1070,1072],{"class":294,"line":1066},18,[292,1068,1069],{"class":488},"  restartPolicy",[292,1071,512],{"class":298},[292,1073,1074],{"class":306},"Never\n",[211,1076,1077],{},"From here I could simply exec shell in and run chown commands on the data volume.",[253,1079,1081],{"id":1080},"conclusion","Conclusion",[211,1083,1084],{},"Mostly smooth. I did get everything back up and running with their original data apart from one app. For some reason -\nthat had a longhorn backup but it was empty. Most likely my fault - some error in the original configuration.",[211,1086,1087],{},"I learnt a lot about PV and PVC deployment.",[211,1089,1090],{},"Longhorn is very easy to work with.",[211,1092,1093],{},"At various points I also learned to play with debug pods and init containers to help with debugging etc.",[211,1095,1096],{},"GitHub copilot (GPT 4.1 in agent mode) probably reduced the time I spent figuring out the changes to about 1\u002F10th of\nwhat it would have been. Still don't feel replaced yet - it still needs direction :)",[1098,1099,1100],"style",{},"html pre.shiki code .s95oV, html code.shiki .s95oV{--shiki-default:#E1E4E8}html pre.shiki code .snl16, html code.shiki .snl16{--shiki-default:#F97583}html pre.shiki code .sU2Wk, html code.shiki .sU2Wk{--shiki-default:#9ECBFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html pre.shiki code .svObZ, html code.shiki .svObZ{--shiki-default:#B392F0}html pre.shiki code .sDLfK, html code.shiki .sDLfK{--shiki-default:#79B8FF}html pre.shiki code .s4JwU, html code.shiki .s4JwU{--shiki-default:#85E89D}",{"title":287,"searchDepth":495,"depth":495,"links":1102},[1103,1109,1110],{"id":255,"depth":495,"text":256,"children":1104},[1105,1106,1107,1108],{"id":260,"depth":506,"text":261},{"id":318,"depth":506,"text":319},{"id":440,"depth":506,"text":441},{"id":469,"depth":506,"text":470},{"id":893,"depth":495,"text":894},{"id":1080,"depth":495,"text":1081},null,"2025-05-17 09:20 +0200","I run a little k3s cluster on a couple of Raspberry Pi 5s and an old intel nuc at home.","md","The hard disc on the control-plane died and after rebuilding the cluster I needed to recover longhorn backups - how?",{},true,"\u002F2025\u002F05\u002F17\u002Frecovering-longhorn-backups-after-a-k3s-control-plane-crash",{"title":206,"description":1113},{"loc":1118},"2025\u002F05\u002F17\u002Frecovering-longhorn-backups-after-a-k3s-control-plane-crash",[223,1123,1124,1125,235],"longhorn","backup","kubernetes","jYo8fBoRKwPiwTexv9dwAik_ic-Xd_LyooVFokMbmhg",1775293006384]