Merge pull request #1509 from xodio/fix-1506-silent-update-index

Handle package indexes update fails
This commit is contained in:
Kirill Shumilov
2018-11-02 16:39:30 +03:00
committed by GitHub
13 changed files with 239 additions and 118 deletions

View File

@@ -73,17 +73,8 @@ Returns a list of all boards that found in all `package_*_index.json` files.
- Returns `Promise<Array<AvailableBoard>>`
### addPackageIndexUrl(url)
Adds an additional package index URL to the config file.
Later you can download and install the package with `arduino-cli` (call `core.updateIndex()`).
Accepts:
- `url` `<String>` — a URL of the third-party `package_*_index.json` file
- Returns `Promise<String>` with the just added URL
### addPackageIndexUrls(urls)
Adds a list of additional package index urls into the config file.
### setPackageIndexUrls(urls)
Sets a list of additional package index urls into the config file.
Later you can download and install packages with `arduino-cli` (call `core.updateIndex()`).
Accepts:

View File

@@ -4,11 +4,15 @@ import { resolve } from 'path';
import * as fse from 'fs-extra';
import YAML from 'yamljs';
export const ADDITIONAL_URLS_PATH = ['board_manager', 'additional_urls'];
const getDefaultConfig = configDir => ({
sketchbook_path: resolve(configDir, 'sketchbook'),
arduino_data: resolve(configDir, 'data'),
});
const stringifyConfig = cfg => YAML.stringify(cfg, 10, 2);
// :: Path -> Object -> { config: Object, path: Path }
export const saveConfig = (configPath, config) => {
const yamlString = YAML.stringify(config, 2);
@@ -35,25 +39,11 @@ export const configure = inputConfig => {
};
// :: Path -> [URL] -> Promise [URL] Error
export const addPackageIndexUrls = (configPath, urls) =>
export const setPackageIndexUrls = (configPath, urls) =>
fse
.readFile(configPath, { encoding: 'utf8' })
.then(YAML.parse)
.then(
R.over(
R.lensPath(['board_manager', 'additional_urls']),
R.pipe(
R.defaultTo([]),
R.concat(R.__, urls),
R.reject(R.isEmpty),
R.uniq
)
)
)
.then(cfg => YAML.stringify(cfg, 10, 2))
.then(R.assocPath(ADDITIONAL_URLS_PATH, urls))
.then(stringifyConfig)
.then(data => fse.writeFile(configPath, data))
.then(R.always(urls));
// :: Path -> URL -> Promise URL Error
export const addPackageIndexUrl = (configPath, url) =>
addPackageIndexUrls(configPath, [url]).then(R.always(url));

View File

@@ -5,12 +5,7 @@ import { exec, spawn } from 'child-process-promise';
import YAML from 'yamljs';
import { remove } from 'fs-extra';
import {
saveConfig,
configure,
addPackageIndexUrl,
addPackageIndexUrls,
} from './config';
import { saveConfig, configure, setPackageIndexUrls } from './config';
import { patchBoardsWithOptions } from './optionParser';
import listAvailableBoards from './listAvailableBoards';
import parseProgressLog from './parseProgressLog';
@@ -113,7 +108,7 @@ const ArduinoCli = (pathToBin, config = null) => {
},
listConnectedBoards: () => listBoardsWith('list', R.prop('serialBoards')),
listInstalledBoards: () => listBoardsWith('listall', R.prop('boards')),
listAvailableBoards: () => listAvailableBoards(cfg.arduino_data),
listAvailableBoards: () => listAvailableBoards(getConfig, cfg.arduino_data),
compile: (onProgress, fqbn, sketchName, verbose = false) =>
runWithProgress(
onProgress,
@@ -164,8 +159,7 @@ const ArduinoCli = (pathToBin, config = null) => {
run(`sketch new ${sketchName}`).then(
R.always(resolve(cfg.sketchbook_path, sketchName, `${sketchName}.ino`))
),
addPackageIndexUrl: url => addPackageIndexUrl(configPath, url),
addPackageIndexUrls: urls => addPackageIndexUrls(configPath, urls),
setPackageIndexUrls: urls => setPackageIndexUrls(configPath, urls),
};
};

View File

@@ -14,24 +14,35 @@
*/
import path from 'path';
import { parse } from 'url';
import * as R from 'ramda';
import * as fse from 'fs-extra';
import versionCompare from 'tiny-version-compare';
import { ADDITIONAL_URLS_PATH } from './config';
const ORIGINAL_PACKAGE_INDEX_FILE = 'package_index.json';
// AvailableBoard :: { name :: String, package :: String }
/**
* Finds all package index json files in the specified directory.
* Returns a promise with a list of full paths to the json files.
* Returns a list of paths to the additional package index files.
*
* :: Path -> Promise [Path] Error
* Gets filenames of additional package index files from arduino cli config
* by parsing URLs and joins filenames with path to packages directory.
*
* :: (() -> Promise Object Error) -> Path -> Promise [Path] Error
*/
const findPackageIndexFiles = dir =>
R.composeP(
R.map(fname => path.join(dir, fname)),
R.filter(R.test(/^package_(.*_)?index.json$/)),
fse.readdir
)(dir);
const getPackageIndexFiles = async (getConfig, packagesDir) => {
const config = await getConfig();
const urls = R.pathOr([], ADDITIONAL_URLS_PATH, config);
const filepaths = R.compose(
R.map(fname => path.join(packagesDir, fname)),
R.append(ORIGINAL_PACKAGE_INDEX_FILE),
R.map(R.compose(R.last, R.split('/'), R.prop('pathname'), parse))
)(urls);
return filepaths;
};
/**
* Reads package index json files, take all package object from them and
@@ -77,10 +88,10 @@ const getAvailableBoards = R.compose(
* Reads all package index json files in the specified directory
* and returns a promise with a list of Available Boards.
*
* :: Path -> Promise [AvailableBoard] Error
* :: (() -> Promise Object Error) -> Path -> Promise [AvailableBoard] Error
*/
export default R.composeP(
getAvailableBoards,
readPackages,
findPackageIndexFiles
getPackageIndexFiles
);

View File

@@ -72,14 +72,14 @@ describe('Arduino Cli', () => {
it('adds URL into .cli-config.yml', () => {
const cli = arduinoCli(PATH_TO_CLI, cfg);
return cli
.addPackageIndexUrl(url)
.setPackageIndexUrls([url])
.then(() => cli.dumpConfig())
.then(res => assert.include(res.board_manager.additional_urls, url));
});
it('downloads additional package index', () => {
const cli = arduinoCli(PATH_TO_CLI, cfg);
return cli
.addPackageIndexUrl(url)
.setPackageIndexUrls([url])
.then(() => cli.core.updateIndex())
.then(() =>
fse.pathExists(

View File

@@ -195,7 +195,12 @@ describe('listAvailableBoards()', () => {
];
it('Lists boards parsed from two package index files', () =>
listAvailableBoards(fixturesDir).then(res =>
assert.sameDeepMembers(res, boards)
));
listAvailableBoards(
() => ({
board_manager: {
additional_urls: ['http://test.com/package_esp8266com_index.json'],
},
}),
fixturesDir
).then(res => assert.sameDeepMembers(res, boards)));
});

View File

@@ -5,7 +5,7 @@ import * as R from 'ramda';
import * as fse from 'fs-extra';
import arduinoCli from 'arduino-cli';
import * as xd from 'xod-deploy';
import { createError, isAmong } from 'xod-func-tools';
import { createError } from 'xod-func-tools';
import * as cpx from 'cpx';
import subscribeIpc from './subscribeIpc';
@@ -79,6 +79,9 @@ const getArduinoCliPath = () =>
const getLibsDir = p => path.join(p, ARDUINO_LIBRARIES_DIRNAME);
// :: String -> [String]
const parseExtraTxtContent = R.compose(R.reject(R.isEmpty), R.split(/\r\n|\n/));
// :: Path -> Path -> Promise Path -> Error
const copy = async (from, to) =>
new Promise((resolve, reject) => {
@@ -97,14 +100,31 @@ const copyLibraries = async (bundledLibDir, userLibDir, sketchbookLibDir) => {
return sketchbookLibDir;
};
// :: Path -> Path
const getExtraTxtPath = wsPath =>
path.join(wsPath, ARDUINO_PACKAGES_DIRNAME, ARDUINO_EXTRA_URLS_FILENAME);
// :: Path -> Promise Path Error
const ensureExtraTxt = async wsPath => {
const extraTxtFilePath = path.join(
wsPath,
ARDUINO_PACKAGES_DIRNAME,
ARDUINO_EXTRA_URLS_FILENAME
);
await fse.ensureFile(extraTxtFilePath);
const extraTxtFilePath = getExtraTxtPath(wsPath);
const doesExist = await fse.pathExists(extraTxtFilePath);
if (!doesExist) {
const bundledUrls = R.join('\n', BUNDLED_ADDITIONAL_URLS);
await fse.writeFile(extraTxtFilePath, bundledUrls, { flag: 'wx' });
} else {
// TODO: For Users on 0.25.0 or 0.25.1 we have to add bundled esp8266
// into existing `extra.txt`. One day we'll remove this kludge.
const extraTxtContents = await fse.readFile(extraTxtFilePath, {
encoding: 'utf8',
});
const extraUrls = parseExtraTxtContent(extraTxtContents);
const newContents = R.compose(
R.join('\n'),
R.concat(R.__, extraUrls),
R.difference(BUNDLED_ADDITIONAL_URLS)
)(extraUrls);
await fse.writeFile(extraTxtFilePath, newContents);
}
return extraTxtFilePath;
};
@@ -293,6 +313,22 @@ const prepareWorkspacePackagesDir = async wsPath => {
return packagesDirPath;
};
/**
* Copies URLs to additional package index files from `extra.txt` into
* `arduino-cli` config file.
*
* :: Path -> ArduinoCli -> Promise [URL] Error
*/
const syncAdditionalPackages = async (wsPath, cli) => {
const extraTxtPath = getExtraTxtPath(wsPath);
const extraTxtContent = await fse.readFile(extraTxtPath, {
encoding: 'utf8',
});
const urls = parseExtraTxtContent(extraTxtContent);
return cli.setPackageIndexUrls(urls);
};
/**
* Creates an instance of ArduinoCli.
*
@@ -317,13 +353,14 @@ export const create = async sketchDir => {
});
}
return arduinoCli(arduinoCliPath, {
const cli = arduinoCli(arduinoCliPath, {
arduino_data: packagesDirPath,
sketchbook_path: sketchDir,
board_manager: {
additional_urls: BUNDLED_ADDITIONAL_URLS,
},
});
await syncAdditionalPackages(wsPath, cli);
return cli;
};
/**
@@ -339,30 +376,33 @@ export const switchWorkspace = async (cli, newWsPath) => {
const oldConfig = await cli.dumpConfig();
const packagesDirPath = await prepareWorkspacePackagesDir(newWsPath);
const newConfig = R.assoc('arduino_data', packagesDirPath, oldConfig);
return cli.updateConfig(newConfig);
const result = cli.updateConfig(newConfig);
await syncAdditionalPackages(newWsPath, cli);
return result;
};
/**
* Updates package index json files.
* Returns log of updating.
* It updates pacakge index files or throw an error.
* Function for internal use only.
*
* :: ArduinoCli -> Promise String Error
* Needed as a separate function to avoid circular function dependencies:
* `listBoards` and `updateIndexes`
*
* It could fail when:
* - no internet connection
* - host not found
*
* :: Path -> ArduinoCli -> Promise _ Error
*/
const updateIndexes = async cli => {
const urls = await cli
.dumpConfig()
.then(R.pathOr([], ['board_manager', 'additional_urls']));
return R.composeP(
() => cli.core.updateIndex(),
cli.addPackageIndexUrls,
R.reject(isAmong(urls)),
R.split('\r\n'),
p => fse.readFile(p, { encoding: 'utf8' }),
ensureExtraTxt,
loadWorkspacePath
)();
};
const updateIndexesInternal = (wsPath, cli) =>
cli.core.updateIndex().catch(err => {
throw createError('UPDATE_INDEXES_ERROR_NO_CONNECTION', {
pkgPath: getArduinoPackagesPath(wsPath),
// `arduino-cli` outputs everything in stdout
// so we have to extract only errors from stdout:
error: R.replace(/^(.|\s)+(?=Error:)/gm, '', err.stdout),
});
});
/**
* Returns map of installed boards and boards that could be installed:
@@ -373,13 +413,62 @@ const updateIndexes = async cli => {
*
* :: ArduinoCli -> Promise { installed :: [InstalledBoard], available :: [AvailableBoard] } Error
*/
export const listBoards = async cli =>
Promise.all([cli.listInstalledBoards(), cli.listAvailableBoards()]).then(
res => ({
export const listBoards = async cli => {
const wsPath = await loadWorkspacePath();
await syncAdditionalPackages(wsPath, cli);
return Promise.all([
cli.listInstalledBoards().catch(err => {
const errContents = JSON.parse(err.stdout);
const normalizedError = new Error(errContents.Cause);
normalizedError.code = err.code;
throw normalizedError;
}),
cli.listAvailableBoards(),
])
.then(res => ({
installed: res[0],
available: res[1],
})
);
}))
.catch(async err => {
if (R.propEq('code', 6, err)) {
// Catch error produced by arduino-cli, but actually it's not an error:
// When User added a new URL into `extra.txt` file it causes that
// arduino-cli tries to read new JSON but it's not existing yet
// so it fails with error "no such file or directory"
// To avoid this and make a good UX, we'll force call `updateIndexes`
return updateIndexesInternal(wsPath, cli).then(() => listBoards(cli));
}
throw createError('UPDATE_INDEXES_ERROR_BROKEN_FILE', {
pkgPath: getArduinoPackagesPath(await loadWorkspacePath()),
error: err.message,
});
});
};
/**
* Updates package index json files.
* Returns a list of just added URLs
*
* :: ArduinoCli -> Promise [URL] Error
*/
const updateIndexes = async cli => {
const wsPath = await loadWorkspacePath();
const addedUrls = await syncAdditionalPackages(wsPath, cli);
await updateIndexesInternal(wsPath, cli);
// We have to call `listBoards` to be sure
// all new index files are valid, because `updateIndex`
// only downloads index files without validating
// Bug reported: https://github.com/arduino/arduino-cli/issues/81
await listBoards(cli);
return addedUrls;
};
/**
* Saves code into arduino-cli sketchbook directory.

View File

@@ -41,7 +41,7 @@ const checkArduinoPackageInstalled = async (cli, packages) => {
};
// :: (ProgressData -> _) -> ArduinoCli -> [{ package, packageName }] -> Promise [{ package, packageName, installed }] Error
const installArduinoPackages = async (onProgress, cli, packages) =>
const installArduinoPackages = (onProgress, cli, packages) =>
Promise.all(
R.map(pkg => cli.core.install(onProgress, pkg.package))(packages)
).then(() => R.map(R.assoc('installed', true))(packages));

View File

@@ -8,6 +8,8 @@ import { ARDUPACKAGES_UPGRADE_PROCEED } from './actionTypes';
import MSG from './messages';
import getLibraryNames from './getLibraryNames';
import { formatErrorMessage, formatLogError } from '../view/formatError';
const progressToProcess = R.curry((processFn, progressData) => {
processFn(progressData.message, progressData.percentage);
});
@@ -43,7 +45,12 @@ export default store => next => action => {
);
proc.success();
})
.catch(err => proc.fail(err.message, 0));
.catch(err => {
const snackbarError = formatErrorMessage(err);
const logErr = formatLogError(err);
store.dispatch(client.addError(snackbarError));
proc.fail(logErr, 0);
});
},
maybeData
);
@@ -61,7 +68,12 @@ export default store => next => action => {
);
proc.success();
})
.catch(err => proc.fail(err.message, 0));
.catch(err => {
const snackbarError = formatErrorMessage(err);
const logErr = formatLogError(err);
store.dispatch(client.addError(snackbarError));
proc.fail(logErr, 0);
});
}
return next(action);

View File

@@ -124,7 +124,8 @@ class PopupUploadConfig extends React.Component {
if (!isBoardSelected || !doesSelectedBoardExist) {
this.changeBoard(defaultBoardIndex);
}
});
})
.catch(this.props.onError);
}
getPorts() {
@@ -177,7 +178,10 @@ class PopupUploadConfig extends React.Component {
this.setState({ boards: null });
updateIndexFiles()
.then(() => this.getBoards())
.catch(() => this.setState({ boards: oldBoards }));
.catch(err => {
this.props.onError(err);
this.setState({ boards: oldBoards });
});
}
changeBoard(boardIndex) {
@@ -423,6 +427,7 @@ PopupUploadConfig.propTypes = {
onPortChanged: PropTypes.func,
onUpload: PropTypes.func,
onClose: PropTypes.func,
onError: PropTypes.func,
};
PopupUploadConfig.defaultProps = {

View File

@@ -19,4 +19,14 @@ export default {
UPLOADED_SUCCESSFULLY: () => ({
title: 'Uploaded successfully',
}),
UPDATE_INDEXES_ERROR_BROKEN_FILE: ({ pkgPath, error }) => ({
title: 'Package index broken',
note: `Error: ${error}`,
solution: `Check correctness of the corresponding URL in "${pkgPath}/extra.txt" and try to update indexes again`,
}),
UPDATE_INDEXES_ERROR_NO_CONNECTION: ({ pkgPath, error }) => ({
title: 'Cannot update indexes',
note: error,
solution: `Check your internet connection and correctness of URLs in "${pkgPath}/extra.txt", then try again`,
}),
};

View File

@@ -9,14 +9,12 @@ import isDevelopment from 'electron-is-dev';
import { ipcRenderer, remote as remoteElectron, shell } from 'electron';
import client from 'xod-client';
import { Project, getProjectName, messages as xpMessages } from 'xod-project';
import { messages as xdMessages } from 'xod-deploy';
import { Project, getProjectName } from 'xod-project';
import {
foldEither,
isAmong,
explodeMaybe,
noop,
composeErrorFormatters,
tapP,
eitherToPromise,
createError,
@@ -28,6 +26,7 @@ import packageJson from '../../../package.json';
import * as actions from '../actions';
import * as uploadActions from '../../upload/actions';
import { listBoards, upload } from '../../upload/arduinoCli';
import uploadMessages from '../../upload/messages';
import * as debuggerIPC from '../../debugger/ipcActions';
import {
getUploadProcess,
@@ -42,7 +41,6 @@ import { SaveProgressBar } from '../components/SaveProgressBar';
import formatError from '../../shared/errorFormatter';
import * as EVENTS from '../../shared/events';
import { default as arduinoDepMessages } from '../../arduinoDependencies/messages';
import { INSTALL_ARDUINO_DEPENDENCIES_MSG } from '../../arduinoDependencies/constants';
import {
checkDeps,
@@ -51,7 +49,6 @@ import {
proceedPackageUpgrade,
} from '../../arduinoDependencies/actions';
import { createSystemMessage } from '../../shared/debuggerMessages';
import uploadMessages from '../../upload/messages';
import getLibraryNames from '../../arduinoDependencies/getLibraryNames';
@@ -68,17 +65,12 @@ import { STATES, getEventNameWithState } from '../../shared/eventStates';
import UpdateArduinoPackagesPopup from '../../arduinoDependencies/components/UpdateArduinoPackagesPopup';
import { checkArduinoDependencies } from '../../arduinoDependencies/runners';
import { formatErrorMessage, formatLogError } from '../formatError';
const { app, dialog, Menu } = remoteElectron;
const DEFAULT_CANVAS_WIDTH = 800;
const DEFAULT_CANVAS_HEIGHT = 600;
const formatErrorMessage = composeErrorFormatters([
xpMessages,
xdMessages,
arduinoDepMessages,
uploadMessages,
]);
const defaultState = {
size: client.getViewableSize(DEFAULT_CANVAS_WIDTH, DEFAULT_CANVAS_HEIGHT),
workspace: '',
@@ -122,6 +114,8 @@ class App extends client.App {
this.onLoadProject = this.onLoadProject.bind(this);
this.onArduinoPathChange = this.onArduinoPathChange.bind(this);
this.showError = this.showError.bind(this);
this.hideAllPopups = this.hideAllPopups.bind(this);
this.showPopupSetWorkspace = this.showPopupSetWorkspace.bind(this);
this.showPopupSetWorkspaceNotCancellable = this.showPopupSetWorkspaceNotCancellable.bind(
@@ -156,7 +150,7 @@ class App extends client.App {
this.showCreateWorkspacePopup(path, force)
);
ipcRenderer.on(EVENTS.WORKSPACE_ERROR, (event, error) => {
this.props.actions.addError(formatErrorMessage(error));
this.showError(error);
});
ipcRenderer.on(EVENTS.REQUEST_CLOSE_WINDOW, () => {
this.confirmUnsavedChanges(() => {
@@ -175,7 +169,7 @@ class App extends client.App {
// Notify about errors in the Main Process
ipcRenderer.on(EVENTS.ERROR_IN_MAIN_PROCESS, (event, error) => {
console.error(error); // eslint-disable-line no-console
this.props.actions.addError(formatErrorMessage(error));
this.showError(error);
});
this.urlActions = {
@@ -270,16 +264,8 @@ class App extends client.App {
const eitherTProject = this.transformProjectForTranspiler(debug);
const logError = logProcessFn => error => {
const stanza = formatErrorMessage(error);
const messageForConsole = [
...(stanza.title ? [stanza.title] : []),
...(stanza.path ? [stanza.path.join(' -> ')] : []),
...(stanza.note ? [stanza.note] : []),
...(stanza.solution ? [stanza.solution] : []),
].join('\n');
logProcessFn(messageForConsole, 0);
};
const logError = logProcessFn => error =>
logProcessFn(formatLogError(error), 0);
stopDebuggerSession();
@@ -476,6 +462,10 @@ class App extends client.App {
);
}
showError(error) {
this.props.actions.addError(formatErrorMessage(error));
}
confirmUnsavedChanges(onConfirm) {
if (!this.props.hasUnsavedChanges) {
onConfirm();
@@ -799,7 +789,7 @@ class App extends client.App {
if (installationNeeded) {
const err = getError(libsToInstall, packagesToInstall);
this.props.actions.addError(
this.props.actions.addNotification(
formatErrorMessage(err),
INSTALL_ARDUINO_DEPENDENCIES_MSG
);
@@ -828,6 +818,7 @@ class App extends client.App {
onPortChanged={this.onSerialPortChange}
onUpload={this.onUploadToArduino}
onClose={this.onUploadConfigClose}
onError={this.showError}
/>
) : null;
}

View File

@@ -0,0 +1,23 @@
import { composeErrorFormatters } from 'xod-func-tools';
import { messages as xpMessages } from 'xod-project';
import { messages as xdMessages } from 'xod-deploy';
import { default as arduinoDepMessages } from '../arduinoDependencies/messages';
import uploadMessages from '../upload/messages';
export const formatErrorMessage = composeErrorFormatters([
xpMessages,
xdMessages,
arduinoDepMessages,
uploadMessages,
]);
export const formatLogError = error => {
const stanza = formatErrorMessage(error);
return [
...(stanza.title ? [stanza.title] : []),
...(stanza.path ? [stanza.path.join(' -> ')] : []),
...(stanza.note ? [stanza.note] : []),
...(stanza.solution ? [stanza.solution] : []),
].join('\n');
};