Oops, read PLL1 documentation again. Looks like FBSEL=1 is for "normal operation". So include that, but use DIRECT=1 to skip the PSEL divider (which would prevent us producing 204MHz from an in-spec PLL frequency).

firmware/common/hackrf_core.c

@@ -252,6 +252,7 @@ void cpu_clock_init(void)
             | CGU_PLL1_CTRL_PSEL(0)
             | CGU_PLL1_CTRL_NSEL(0)
 			| CGU_PLL1_CTRL_MSEL(16)
+			| CGU_PLL1_CTRL_FBSEL
 			| CGU_PLL1_CTRL_DIRECT;
 
 	/* wait until stable */